{"meta":{"date":"2026-09-21","timezone":"UTC","generated_at":"2026-10-14T22:13:33Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":92,"failed_runs":89,"runs_with_occurrences":89,"runs_with_failure_pattern_matches":80,"runs_with_unmatched_occurrences":9,"runs_without_occurrence_rows":3,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102182452419104768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6359,"pr_state":"open","pr_sha":"ec8ebc721039ebe68bd095d89ea7760a1a74f924","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T23:45:27Z","started_at":"2026-09-21T23:45:27Z","completed_at":"2026-09-22T01:44:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T01:46:13.86821453Z","timing_metadata_checked_at":"2026-09-22T01:46:13.86821453Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T01:46:13.873069414Z","region_metadata_checked_at":"2026-09-22T01:46:13.873069414Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"120fca1a4f6b351ea3668c36a8cbfd8c9953880b9ce4fbcf6cfb427474529cb1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102182452419104768","occurred_at":"2026-09-21T23:45:27Z","failed_at":"alert","signature_id":"c22812e556a5519ae937ff6a1730e7cdeeaa3ba3ad47e15b9bf73b05748af1b0","test_name":"[aro-hcp-observability] [hcp] alert userJourneySwiftRouterAvailability1h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-22T00:47:01Z\n  Ended:       2026-09-22T01:21:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneySwiftRouterAvailability1h5m\", burn_rate_tier=\"fast\", cluster=\"ci01-j9104768-mgmt-1\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9104768/providers/microsoft.monitor/accounts/hcps-j9104768\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"swift-router-availability\"\n  Description: SWIFT router-capacity availability is consuming its error budget at the fast burn rate.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"27efdba0688c010a9407237cceb9d29c447c7f9bd7965cf5f46a9ce24313bf72","failure_pattern":"alert [hcp] userJourneySwiftRouterAvailability1h5m fired","search_query":"alert [hcp] userJourneySwiftRouterAvailability1h5m fired"}},{"row_id":"814d27218307d7ad22811307c139ab61817919d6fd20915c3bf1d642e7553fe7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102182452419104768","occurred_at":"2026-09-21T23:45:27Z","failed_at":"alert","signature_id":"221ef79e0877fe1cc3d6cf895915aca4f68534f6d76955ac475aad76770c339d","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-22T00:44:59Z\n  Ended:       2026-09-22T01:17:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j9104768-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j9104768-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"ab5f5a6411828227d2805d6a025f25304876292d3a8bbb51389dc51557b65dad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102182452419104768","occurred_at":"2026-09-21T23:45:27Z","failed_at":"alert","signature_id":"42d58e44fe5c74f647d9e0212abbeb90537447a99640fc4f5c3e2ccf1c325665","test_name":"[aro-hcp-observability] [hcp] alert userJourneySwiftRouterAvailability6h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-22T01:26:03Z\n  Ended:       2026-09-22T01:32:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneySwiftRouterAvailability6h30m\", burn_rate_tier=\"medium\", cluster=\"ci01-j9104768-mgmt-2\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9104768/providers/microsoft.monitor/accounts/hcps-j9104768\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"swift-router-availability\"\n  Description: SWIFT router-capacity availability is consuming its error budget at the medium burn rate.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7fa847820ad43185d68e54ccb17caf15404b33f7f9865c668f2186c67969315a","failure_pattern":"alert [hcp] userJourneySwiftRouterAvailability6h30m fired","search_query":"alert [hcp] userJourneySwiftRouterAvailability6h30m fired"}},{"row_id":"c137e554b492d1086c0d7e0f60d04e0bbe72284c00d7480258a6523682d0ad1d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102182452419104768","occurred_at":"2026-09-21T23:45:27Z","failed_at":"alert","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-22T00:51:04Z\n  Ended:       2026-09-22T01:02:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9104768-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"cdece7a3be8442ef27c45aff45c15811fb99e088c938fb3c7fd53320c4f4b23b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102182452419104768","occurred_at":"2026-09-21T23:45:27Z","failed_at":"e2e","signature_id":"fa21977665ce9798e9e927898519ad9d0d3cb05d9b569b7a23e923f4fa807513","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-98vcbn\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001510f00\u003e: \n    DNS for route host agnhost-e2e-sample-app-cwflx.apps.aro.m5g7y3m1z1w6o1g.oqts.j9104768.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-cwflx.apps.aro.m5g7y3m1z1w6o1g.oqts.j9104768.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cwflx.apps.aro.m5g7y3m1z1w6o1g.oqts.j9104768.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-98vcbn\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001510f00\u003e: \n    DNS for route host agnhost-e2e-sample-app-cwflx.apps.aro.m5g7y3m1z1w6o1g.oqts.j9104768.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-cwflx.apps.aro.m5g7y3m1z1w6o1g.oqts.j9104768.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-cwflx.apps.aro.m5g7y3m1z1w6o1g.oqts.j9104768.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-28d65.apps.aro.o4p4d0n2f5l6i5t.cztj.j0999424.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-28d65.apps.aro.o4p4d0n2f5l6i5t.cztj.j0999424.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-28d65.apps.aro.o4p4d0n2f5l6i5t.cztj.j0999424.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","27efdba0688c010a9407237cceb9d29c447c7f9bd7965cf5f46a9ce24313bf72","7fa847820ad43185d68e54ccb17caf15404b33f7f9865c668f2186c67969315a","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102180800429559808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6969,"pr_state":"open","pr_sha":"e2568d6729d15e613ed95c09e7245845d5258098","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-21T23:38:54Z","started_at":"2026-09-21T23:38:54Z","completed_at":"2026-09-22T01:45:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T01:46:14.009674787Z","timing_metadata_checked_at":"2026-09-22T01:46:14.009674787Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T01:46:14.012191579Z","region_metadata_checked_at":"2026-09-22T01:46:14.012191579Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7042/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102174018562953216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7042,"pr_state":"closed","pr_sha":"ed561d7657bd4aae970254bd806604f0b5a59888","final_merged_sha":"ed561d7657bd4aae970254bd806604f0b5a59888","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T23:11:57Z","started_at":"2026-09-21T23:11:57Z","completed_at":"2026-09-21T23:39:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T23:50:51.257939088Z","timing_metadata_checked_at":"2026-09-21T23:50:51.257939088Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T23:50:51.261207078Z","region_metadata_checked_at":"2026-09-21T23:50:51.261207078Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"e1a9b78ccac1b27ad6b07156a851915eb29225e6e73d744cb33198740ba299b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7042/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102174018562953216","occurred_at":"2026-09-21T23:11:57Z","failed_at":"provision","signature_id":"51fa2ffb12b10634911880f966fe2309f008fe85ba92ea9da200b0c7ecccdef7","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/admin-api-cert","test_suite":"templatize-pipeline","failure_text":"time=2026-09-21T23:34:09.997Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert description=\"Step admin-api-cert\\n Kind: CreateCertificate\\n\"\ntime=2026-09-21T23:34:09.998Z level=INFO msg=\"Creating certificate\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert vaultBaseUrl=https://aro-hcp-dev-svc-kv.vault.azure.net/ certificateName=admin-api-cert-ci01-j2953216 issuer=Self commonName=admin.j2953216.hcpsvc.osadev.cloud san=admin.j2953216.hcpsvc.osadev.cloud\ntime=2026-09-21T23:34:10.858Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert err=\"error running Create Certificate Step: failed to create certificate \\\"admin-api-cert-ci01-j2953216\\\" in vault \\\"https://aro-hcp-dev-svc-kv.vault.azure.net/\\\": POST https://aro-hcp-dev-svc-kv.vault.azure.net/certificates/admin-api-cert-ci01-j2953216/create\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Certificate admin-api-cert-ci01-j2953216 is currently in a deleted but recoverable state, and its name cannot be reused; in this state, the certificate can only be recovered or purged.\\\",\\n    \\\"innererror\\\": {\\n      \\\"code\\\": \\\"ObjectIsDeletedButRecoverable\\\"\\n    }\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6f6e7e66ad8bc42bd6647f1e246b1e77205441e0c9495974d9b1f940d8bdec9c","failure_pattern":"ERROR CODE: Conflict; detail code ObjectIsDeletedButRecoverable","search_query":"ERROR CODE: Conflict"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6f6e7e66ad8bc42bd6647f1e246b1e77205441e0c9495974d9b1f940d8bdec9c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7057/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102158779100434432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7057,"pr_state":"closed","pr_sha":"c2ceaf7839e32693059551ef02987308dc74b730","final_merged_sha":"c2ceaf7839e32693059551ef02987308dc74b730","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T22:11:23Z","started_at":"2026-09-21T22:11:23Z","completed_at":"2026-09-22T00:41:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T00:43:47.028244546Z","timing_metadata_checked_at":"2026-09-22T00:43:47.028244546Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T00:43:47.031085737Z","region_metadata_checked_at":"2026-09-22T00:43:47.031085737Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"0e8db1c2b082ef37dccca8e8f94386764776d19f3d06acc05d61f4b3bca1f3c2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7057/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102158779100434432","occurred_at":"2026-09-21T22:11:23Z","failed_at":"e2e","signature_id":"cac3f38be94e3ba04b3f8927cd615e2482d23135bd2a91751651c57c9c269d9c","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f3a090\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-9kb6b8kvhn7s\" to finish deleting: GET https://rp.j0434432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/825dbd39-48b3-4fef-8b8e-49148be72c72\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/825dbd39-48b3-4fef-8b8e-49148be72c72\",\n      \"name\": \"825dbd39-48b3-4fef-8b8e-49148be72c72\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T23:21:12.831453866Z\",\n      \"endTime\": \"2026-09-21T23:45:12.878649358Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svmegsg25m4rvg6v9g6bgt2bkfpnu3j still exists (deletion dispatched at 2026-09-21T23:21:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f3a090\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-9kb6b8kvhn7s\" to finish deleting: GET https://rp.j0434432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/825dbd39-48b3-4fef-8b8e-49148be72c72\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/825dbd39-48b3-4fef-8b8e-49148be72c72\",\n      \"name\": \"825dbd39-48b3-4fef-8b8e-49148be72c72\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T23:21:12.831453866Z\",\n      \"endTime\": \"2026-09-21T23:45:12.878649358Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svmegsg25m4rvg6v9g6bgt2bkfpnu3j still exists (deletion dispatched at 2026-09-21T23:21:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"65c84ce3b013d84f34064f7522da67a2c9ec813f6462729ac07e57150fc8729d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7057/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102158779100434432","occurred_at":"2026-09-21T22:11:23Z","failed_at":"e2e","signature_id":"778d0dd2839c709fa5131013b1b5f5f8d953f7598d191cdfa5104c56f3226107","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000b0e870\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-pg24gmdfnmzw\" to finish deleting: GET https://rp.j0434432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7836cbe1-bcbe-4646-898f-78e819a0d98f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7836cbe1-bcbe-4646-898f-78e819a0d98f\",\n      \"name\": \"7836cbe1-bcbe-4646-898f-78e819a0d98f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T23:35:23.137655617Z\",\n      \"endTime\": \"2026-09-21T23:59:32.105878829Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svmdju7l81957j5gpbargner8e51jae still exists (deletion dispatched at 2026-09-21T23:35:24Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000b0e870\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-pg24gmdfnmzw\" to finish deleting: GET https://rp.j0434432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7836cbe1-bcbe-4646-898f-78e819a0d98f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7836cbe1-bcbe-4646-898f-78e819a0d98f\",\n      \"name\": \"7836cbe1-bcbe-4646-898f-78e819a0d98f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T23:35:23.137655617Z\",\n      \"endTime\": \"2026-09-21T23:59:32.105878829Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svmdju7l81957j5gpbargner8e51jae still exists (deletion dispatched at 2026-09-21T23:35:24Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"df160f249dc3e0dc1e038d42df412a81ab440b2c0335a0efafa4306b7d2e942d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7057/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102158779100434432","occurred_at":"2026-09-21T22:11:23Z","failed_at":"alert","signature_id":"e47bfab8d17dbbd191c3753845d8e72adefc5781cb141e35640d99f8f1135435","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-22T00:14:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j0434432-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.56:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0434432/providers/microsoft.monitor/accounts/services-j0434432\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-9fd595fb9-74n8n\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j0434432-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-22T00:14:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j0434432-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.56:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0434432/providers/microsoft.monitor/accounts/services-j0434432\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-9fd595fb9-74n8n\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j0434432-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","failure_pattern":"alert [svc] KustoLogsDataStale fired","search_query":"alert [svc] KustoLogsDataStale fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102153867507535872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6945,"pr_state":"closed","pr_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","final_merged_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T21:51:52Z","started_at":"2026-09-21T21:51:52Z","completed_at":"2026-09-21T23:48:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T00:01:25.374064278Z","timing_metadata_checked_at":"2026-09-22T00:01:25.374064278Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T00:01:25.377170569Z","region_metadata_checked_at":"2026-09-22T00:01:25.377170569Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f1b6bb2283b6c08ee54713e27c9711d3abe8f3ae293cb57903782a360ba6ed10","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102153867507535872","occurred_at":"2026-09-21T21:51:52Z","failed_at":"e2e","signature_id":"03df022cf2ee732eed23f0f840ad4f8c58e42ed459b01ea2a28814a363ff7d7f","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:349]: Timed out after 305.584s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:334 with:\nkubectl should execute successfully from authorized VM with OIDC token\nUnexpected error:\n    \u003c*errors.errorString | 0xc001028de0\u003e: \n    E0921 23:06:57.092173    2855 memcache.go:381] \"Couldn't get current server API group list\" err=\"the server has asked for the client to provide credentials\"\n    E0921 23:06:57.110667    2855 memcache.go:381] \"Couldn't get current server API group list\" err=\"the server has asked for the client to provide credentials\"\n    E0921 23:06:57.129290    2855 memcache.go:381] \"Couldn't get current server API group list\" err=\"the server has asked for the client to provide credentials\"\n    E0921 23:06:57.149503    2855 memcache.go:381] \"Couldn't get current server API group list\" err=\"the server has asked for the client to provide credentials\"\n    E0921 23:06:57.163374    2855 memcache.go:381] \"Couldn't get current server API group list\" err=\"the server has asked for the client to provide credentials\"\n    error: You must be logged in to the server (the server has asked for the client to provide credentials)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:349]: Timed out after 305.584s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:334 with:\nkubectl should execute successfully from authorized VM with OIDC token\nUnexpected error:\n    \u003c*errors.errorString | 0xc001028de0\u003e: \n    E0921 23:06:57.092173    2855 memcache.go:381] \"Couldn't get current server API group list\" err=\"the server has asked for the client to provide credentials\"\n    E0921 23:06:57.110667    2855 memcache.go:381] \"Couldn't get current server API group list\" err=\"the server has asked for the client to provide credentials\"\n    E0921 23:06:57.129290    2855 memcache.go:381] \"Couldn't get current server API group list\" err=\"the server has asked for the client to provide credentials\"\n    E0921 23:06:57.149503    2855 memcache.go:381] \"Couldn't get current server API group list\" err=\"the server has asked for the client to provide credentials\"\n    E0921 23:06:57.163374    2855 memcache.go:381] \"Couldn't get current server API group list\" err=\"the server has asked for the client to provide credentials\"\n    error: You must be logged in to the server (the server has asked for the client to provide credentials)\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f00cc40d463cd94eb55b63d664af8e764db2b93c677dee62bb06d736dc1a6b13","failure_pattern":"error: You must be logged in to the server (the server has asked for the client to provide credentials)","search_query":"error: You must be logged in to the server (the server has asked for the client to provide credentials)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f00cc40d463cd94eb55b63d664af8e764db2b93c677dee62bb06d736dc1a6b13"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102147905392480256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5846,"pr_state":"closed","pr_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","final_merged_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T21:28:11Z","started_at":"2026-09-21T21:28:11Z","completed_at":"2026-09-21T23:43:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T00:01:25.17279808Z","timing_metadata_checked_at":"2026-09-22T00:01:25.17279808Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T00:01:25.176100971Z","region_metadata_checked_at":"2026-09-22T00:01:25.176100971Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"142760590699bbd4297083514158fb38502ad2804c202af81b5a07fe3e813945","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102147905392480256","occurred_at":"2026-09-21T21:28:11Z","failed_at":"e2e","signature_id":"c889ad461504bec27fe883db129ec1500d2ff6d0c66f2f838625406e455ba267","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0017180a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-fbrtzb4b2fmg\" to finish deleting: GET https://rp.j2480256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1ee0e1ee-81da-48f8-ace9-84f50bd31687\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1ee0e1ee-81da-48f8-ace9-84f50bd31687\",\n      \"name\": \"1ee0e1ee-81da-48f8-ace9-84f50bd31687\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T22:51:36.752291176Z\",\n      \"endTime\": \"2026-09-21T23:15:42.111964576Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svlsuaisdl7tc2iilfomjdj3bidbk7m still exists (deletion dispatched at 2026-09-21T22:51:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0017180a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-fbrtzb4b2fmg\" to finish deleting: GET https://rp.j2480256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1ee0e1ee-81da-48f8-ace9-84f50bd31687\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1ee0e1ee-81da-48f8-ace9-84f50bd31687\",\n      \"name\": \"1ee0e1ee-81da-48f8-ace9-84f50bd31687\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T22:51:36.752291176Z\",\n      \"endTime\": \"2026-09-21T23:15:42.111964576Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svlsuaisdl7tc2iilfomjdj3bidbk7m still exists (deletion dispatched at 2026-09-21T22:51:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5d0952819fe023ea7ae8eb481e801348137aa06929908b6bd0c2b22346ef995e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102147905392480256","occurred_at":"2026-09-21T21:28:11Z","failed_at":"e2e","signature_id":"251f3650bd31fa7b3a66f5d14cedf3a9ea157af5d413c7fa8997374c81d03bef","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c52168\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-pgjxph86z52p\" to finish deleting: GET https://rp.j2480256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50a0df80-c8f5-49a6-a2ef-c644f0089148\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50a0df80-c8f5-49a6-a2ef-c644f0089148\",\n      \"name\": \"50a0df80-c8f5-49a6-a2ef-c644f0089148\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T22:50:27.873366155Z\",\n      \"endTime\": \"2026-09-21T23:14:33.586111357Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svlspsboe6grnibds7i6q05t212qj6b still exists (deletion dispatched at 2026-09-21T22:50:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c52168\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-pgjxph86z52p\" to finish deleting: GET https://rp.j2480256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50a0df80-c8f5-49a6-a2ef-c644f0089148\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50a0df80-c8f5-49a6-a2ef-c644f0089148\",\n      \"name\": \"50a0df80-c8f5-49a6-a2ef-c644f0089148\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T22:50:27.873366155Z\",\n      \"endTime\": \"2026-09-21T23:14:33.586111357Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svlspsboe6grnibds7i6q05t212qj6b still exists (deletion dispatched at 2026-09-21T22:50:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8f5d0dfa06754ef9f2fa755548f1d1054e9bb220d5b7b89c2059d31fa4b03c6c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102147905392480256","occurred_at":"2026-09-21T21:28:11Z","failed_at":"e2e","signature_id":"4527d024355dd6c67a9c2b1233ba75ecc352996c7b983c10c8772e078ea1dacc","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010d8588\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-zmwhfv\" in resourcegroup=\"rg-np-version-upgrade-zmwhfv-9zjwxr8tvcxr\" to finish deleting: GET https://rp.j2480256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28586483-da92-492d-9e86-c68b31311498\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28586483-da92-492d-9e86-c68b31311498\",\n      \"name\": \"28586483-da92-492d-9e86-c68b31311498\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T22:51:39.781708337Z\",\n      \"endTime\": \"2026-09-21T23:15:42.12352473Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svlskunr07a0jib7gmlijsvopjvkvpk still exists (deletion dispatched at 2026-09-21T22:51:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010d8588\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-zmwhfv\" in resourcegroup=\"rg-np-version-upgrade-zmwhfv-9zjwxr8tvcxr\" to finish deleting: GET https://rp.j2480256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28586483-da92-492d-9e86-c68b31311498\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28586483-da92-492d-9e86-c68b31311498\",\n      \"name\": \"28586483-da92-492d-9e86-c68b31311498\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T22:51:39.781708337Z\",\n      \"endTime\": \"2026-09-21T23:15:42.12352473Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svlskunr07a0jib7gmlijsvopjvkvpk still exists (deletion dispatched at 2026-09-21T22:51:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c6197efdfeb12c50a0490a48a96f399dca289cc16a55c8a8c623db6f9d5840f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102147905392480256","occurred_at":"2026-09-21T21:28:11Z","failed_at":"alert","signature_id":"07c2999f516b4cc21214fd0a6dccc22d5c60e5a648667fede7623f8bfc4613ed","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T23:02:39Z\n  Ended:       2026-09-21T23:30:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j2480256-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} has exceeded 250ms over the past 30 minutes (cluster ci01-j2480256-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"f85b9ad8cb070daec0b9a3eb248630c442f1019da34f71ee0377996fa1a02898","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102147905392480256","occurred_at":"2026-09-21T21:28:11Z","failed_at":"alert","signature_id":"afeef2c62e7c85e77774262a0534d466e87d447af4f2ba74cf5afc04634c2dc6","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-21T23:25:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j2480256-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-spg6n\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-spg6n (namespace arobit) on cluster ci01-j2480256-svc has averaged 126.7% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"fe92d73818d14c92bd96e29ffdc54421a9b9b1467dd4a243d5c5cf1e6ee4c4f5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102147905392480256","occurred_at":"2026-09-21T21:28:11Z","failed_at":"e2e","signature_id":"54796250838c2bab2d64eb91a71e4b4ad4f6e5e357197051e3a50789307888c8","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f60120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-6vd7sh65bb2l\" to finish deleting: GET https://rp.j2480256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/004d700d-e62b-4602-a77a-50acbaef271b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/004d700d-e62b-4602-a77a-50acbaef271b\",\n      \"name\": \"004d700d-e62b-4602-a77a-50acbaef271b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T23:00:42.293045796Z\",\n      \"endTime\": \"2026-09-21T23:24:42.15355258Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svlsrl4d5btnal5n73mhn8c5tst1q3d still exists (deletion dispatched at 2026-09-21T23:00:43Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f60120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-6vd7sh65bb2l\" to finish deleting: GET https://rp.j2480256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/004d700d-e62b-4602-a77a-50acbaef271b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/004d700d-e62b-4602-a77a-50acbaef271b\",\n      \"name\": \"004d700d-e62b-4602-a77a-50acbaef271b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T23:00:42.293045796Z\",\n      \"endTime\": \"2026-09-21T23:24:42.15355258Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svlsrl4d5btnal5n73mhn8c5tst1q3d still exists (deletion dispatched at 2026-09-21T23:00:43Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102140947914559488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7044,"pr_state":"closed","pr_sha":"0f625957a86b5fda4999ce65087e506bc2499c89","final_merged_sha":"0f625957a86b5fda4999ce65087e506bc2499c89","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T21:00:10Z","started_at":"2026-09-21T21:00:10Z","completed_at":"2026-09-21T23:05:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T23:09:15.90232307Z","timing_metadata_checked_at":"2026-09-21T23:09:15.90232307Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T23:09:15.905367061Z","region_metadata_checked_at":"2026-09-21T23:09:15.905367061Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"36e16073202ec3cea859ecbc53e6671bfec5b86012b12f507fcb44d366c5906b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102140947914559488","occurred_at":"2026-09-21T21:00:10Z","failed_at":"e2e","signature_id":"4d88b3916f2af8b1dee739517ca580930c72d0ffd344310c9c9d9b37c22fb36f","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:184]: failed to update cluster with new KMS key\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015dccf0\u003e: \n    failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-fm6rvv7nj2n4\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-fm6rvv7nj2n4, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:184]: failed to update cluster with new KMS key\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015dccf0\u003e: \n    failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-fm6rvv7nj2n4\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-fm6rvv7nj2n4, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f570ff2ef05014779bb7489dca295e256a69aead33d6bff8319ed54db32fd9e1","failure_pattern":"UpdateHCPCluster20260901 timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-fm6rvv7nj2n4\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-fm6rvv7nj2n4, error: context deadline exceeded"}},{"row_id":"b24c7784a427d3fbff89d6f342c030dd8a9521814afbf58f0f60880daa696068","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102140947914559488","occurred_at":"2026-09-21T21:00:10Z","failed_at":"alert","signature_id":"78d28171d48e4312713ac44b6850bb594d070a6572be0ccf0ce3228ede3da313","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-21T22:52:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j4559488-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-wmt7h\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-wmt7h (namespace arobit) on cluster ci01-j4559488-svc has averaged 126.9% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","f570ff2ef05014779bb7489dca295e256a69aead33d6bff8319ed54db32fd9e1"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102139231773134848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7053,"pr_state":"open","pr_sha":"1c995b8c43e3ab7b347b1bdb08b62da193b79171","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T20:53:43Z","started_at":"2026-09-21T20:53:43Z","completed_at":"2026-09-21T23:04:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T23:09:18.065762983Z","timing_metadata_checked_at":"2026-09-21T23:09:18.065762983Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T23:09:18.068681975Z","region_metadata_checked_at":"2026-09-21T23:09:18.068681975Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"16acbadc7a7d607b1b0c79dcd64b12765d2d67d810d0f251f78fe7d0093f044b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102139231773134848","occurred_at":"2026-09-21T20:53:43Z","failed_at":"alert","signature_id":"8c59d63807a821b56b81d2596323acab438e854126c4574d183e3ca7c3652c6b","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T22:19:36Z\n  Ended:       2026-09-21T22:27:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3134848-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3134848-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"a64395bfe9c8ffa3c3caaae5114cd827085338d7228c279628f2cd2d3339fb1c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102139231773134848","occurred_at":"2026-09-21T20:53:43Z","failed_at":"e2e","signature_id":"8d4610a8e3720a752379740a78ab92b95fb9bfef502993d7a8c461417406ce7c","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001926240\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-nkfhdk\" in resourcegroup=\"rg-np-skip-minor-nkfhdk-c27mwk7g5pvc\" to finish deleting: GET https://rp.j3134848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d9eea4f7-a0ab-48ba-91c6-c4918c7d5140\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d9eea4f7-a0ab-48ba-91c6-c4918c7d5140\",\n      \"name\": \"d9eea4f7-a0ab-48ba-91c6-c4918c7d5140\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T22:24:14.551479675Z\",\n      \"endTime\": \"2026-09-21T22:48:22.777443835Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svle0ditu90ti60l09h3683n0vfj4ef still exists (deletion dispatched at 2026-09-21T22:24:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001926240\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-nkfhdk\" in resourcegroup=\"rg-np-skip-minor-nkfhdk-c27mwk7g5pvc\" to finish deleting: GET https://rp.j3134848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d9eea4f7-a0ab-48ba-91c6-c4918c7d5140\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d9eea4f7-a0ab-48ba-91c6-c4918c7d5140\",\n      \"name\": \"d9eea4f7-a0ab-48ba-91c6-c4918c7d5140\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T22:24:14.551479675Z\",\n      \"endTime\": \"2026-09-21T22:48:22.777443835Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svle0ditu90ti60l09h3683n0vfj4ef still exists (deletion dispatched at 2026-09-21T22:24:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102137984487788544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"7cf0a28b3379c46112996aea37d6ac4e12d00ece","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T20:48:45Z","started_at":"2026-09-21T20:48:45Z","completed_at":"2026-09-21T22:58:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T23:09:15.760408996Z","timing_metadata_checked_at":"2026-09-21T23:09:15.760408996Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T23:09:15.764843782Z","region_metadata_checked_at":"2026-09-21T23:09:15.764843782Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"37cd4d309491fa0813ae0c0035a5cedb8fb7ef10380ef0f4417057c550b12b41","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102137984487788544","occurred_at":"2026-09-21T20:48:45Z","failed_at":"e2e","signature_id":"65c5005b0f260fc73e5d54013e7768bcdee6e55a8b8431a42fac57676b1d7923","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n    \u003c*errors.joinError | 0xc0015000f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-l48n5s\" in resourcegroup=\"rg-zstream-upgrade-4-21-fzb65fq622vl\" to finish deleting: GET https://rp.j7788544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/845c3e9c-94c0-4eb7-a5ea-da10ab276259\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/845c3e9c-94c0-4eb7-a5ea-da10ab276259\",\n      \"name\": \"845c3e9c-94c0-4eb7-a5ea-da10ab276259\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T22:13:23.034834553Z\",\n      \"endTime\": \"2026-09-21T22:37:32.060039773Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svle2efk8qd8kfcv06s6nrudgn294rs still exists (deletion dispatched at 2026-09-21T22:13:24Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n    \u003c*errors.joinError | 0xc0015000f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-l48n5s\" in resourcegroup=\"rg-zstream-upgrade-4-21-fzb65fq622vl\" to finish deleting: GET https://rp.j7788544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/845c3e9c-94c0-4eb7-a5ea-da10ab276259\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/845c3e9c-94c0-4eb7-a5ea-da10ab276259\",\n      \"name\": \"845c3e9c-94c0-4eb7-a5ea-da10ab276259\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T22:13:23.034834553Z\",\n      \"endTime\": \"2026-09-21T22:37:32.060039773Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svle2efk8qd8kfcv06s6nrudgn294rs still exists (deletion dispatched at 2026-09-21T22:13:24Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"54e8438d8ce583a8bb8d0f2e727388f7eb87117e82496affc29b06dc79a9869e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102137984487788544","occurred_at":"2026-09-21T20:48:45Z","failed_at":"alert","signature_id":"07d846f7431c74ef4c6803f4499aa15ae4c1bcc4f158d67d502057700488fc6d","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T22:19:37Z\n  Ended:       2026-09-21T22:25:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7788544-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7788544-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"886c58bca633ee81cbc8f0fbd7060652bb6da744d64d9bd17f36045bfe648ce2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102137984487788544","occurred_at":"2026-09-21T20:48:45Z","failed_at":"e2e","signature_id":"e73c60ad7e32baef86674d19f65317ee588bff5f88a0d3ecd7f7ae173e0ec2fd","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011aa0f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-7tlvf2\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-86sk84qrfqhq\" to finish deleting: GET https://rp.j7788544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/596846d9-9365-4693-9508-7358e0e71bb2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/596846d9-9365-4693-9508-7358e0e71bb2\",\n      \"name\": \"596846d9-9365-4693-9508-7358e0e71bb2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T22:15:01.058992145Z\",\n      \"endTime\": \"2026-09-21T22:39:02.102835578Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svle6o9uniishaam7inu4r0e8nt58du still exists (deletion dispatched at 2026-09-21T22:15:09Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011aa0f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-7tlvf2\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-86sk84qrfqhq\" to finish deleting: GET https://rp.j7788544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/596846d9-9365-4693-9508-7358e0e71bb2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/596846d9-9365-4693-9508-7358e0e71bb2\",\n      \"name\": \"596846d9-9365-4693-9508-7358e0e71bb2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T22:15:01.058992145Z\",\n      \"endTime\": \"2026-09-21T22:39:02.102835578Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svle6o9uniishaam7inu4r0e8nt58du still exists (deletion dispatched at 2026-09-21T22:15:09Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7050/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102126077554987008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7050,"pr_state":"closed","pr_sha":"cf72712284cb479e9bca6f86952f68b72255a389","final_merged_sha":"cf72712284cb479e9bca6f86952f68b72255a389","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T20:00:58Z","started_at":"2026-09-21T20:00:58Z","completed_at":"2026-09-21T22:12:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T22:17:31.174966061Z","timing_metadata_checked_at":"2026-09-21T22:17:31.174966061Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T22:17:31.17866775Z","region_metadata_checked_at":"2026-09-21T22:17:31.17866775Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"21e78f52f066e1de5a3598256e0653bcb3724edf7835f913b53d195cd9d1b592","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7050/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102126077554987008","occurred_at":"2026-09-21T20:00:58Z","failed_at":"e2e","signature_id":"248b90da6bcb121d6dfa0b7fcb787086c64f647a417408248097e1bf22bb96c4","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0016fa180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-thtz8zk9z88r\" to finish deleting: GET https://rp.j4987008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f55c577-d3bc-4361-b4bb-f88cc3f62210\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f55c577-d3bc-4361-b4bb-f88cc3f62210\",\n      \"name\": \"7f55c577-d3bc-4361-b4bb-f88cc3f62210\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T21:31:10.627385422Z\",\n      \"endTime\": \"2026-09-21T21:55:11.978379368Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svkp5b9427pmplr3tj6p242bs17unt3 still exists (deletion dispatched at 2026-09-21T21:31:13Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0016fa180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-thtz8zk9z88r\" to finish deleting: GET https://rp.j4987008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f55c577-d3bc-4361-b4bb-f88cc3f62210\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f55c577-d3bc-4361-b4bb-f88cc3f62210\",\n      \"name\": \"7f55c577-d3bc-4361-b4bb-f88cc3f62210\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T21:31:10.627385422Z\",\n      \"endTime\": \"2026-09-21T21:55:11.978379368Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svkp5b9427pmplr3tj6p242bs17unt3 still exists (deletion dispatched at 2026-09-21T21:31:13Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f3f855d1b113d2125408190503a989976b1d3bf9174ba780494a68789c76c546","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7050/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102126077554987008","occurred_at":"2026-09-21T20:00:58Z","failed_at":"alert","signature_id":"43607ad02c6153d3e3b1101e149672679da4db64e0ba53fdb3cb06d864fde126","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T21:33:11Z\n  Ended:       2026-09-21T21:39:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4987008-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4987008-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7042/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102125896348471296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7042,"pr_state":"closed","pr_sha":"6a2ceb3234624c8807be9fd61b93ac14e967a5cf","final_merged_sha":"ed561d7657bd4aae970254bd806604f0b5a59888","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T20:00:40Z","started_at":"2026-09-21T20:00:40Z","completed_at":"2026-09-21T22:16:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T22:27:33.140478611Z","timing_metadata_checked_at":"2026-09-21T22:27:33.140478611Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T22:27:33.143326303Z","region_metadata_checked_at":"2026-09-21T22:27:33.143326303Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"0d81bbda786344f9dd6c3cb2f80a4c458f985575bff988458f3207864bf70a95","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7042/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102125896348471296","occurred_at":"2026-09-21T20:00:40Z","failed_at":"alert","signature_id":"d83e7bd226c38328a83242b823cf052568a0283d0569bb6a5a50fec25a24efc4","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T21:30:47Z\n  Ended:       2026-09-21T21:39:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j8471296-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j8471296-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102125835409428480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6965,"pr_state":"closed","pr_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","final_merged_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-21T20:00:29Z","started_at":"2026-09-21T20:00:29Z","completed_at":"2026-09-21T22:02:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T22:07:11.418105858Z","timing_metadata_checked_at":"2026-09-21T22:07:11.418105858Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T22:07:11.421097249Z","region_metadata_checked_at":"2026-09-21T22:07:11.421097249Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102125829403185152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6969,"pr_state":"open","pr_sha":"c2c1283897768eac520c16e15540fd345ab5deab","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T20:00:27Z","started_at":"2026-09-21T20:00:27Z","completed_at":"2026-09-21T22:17:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T22:27:32.912368272Z","timing_metadata_checked_at":"2026-09-21T22:27:32.912368272Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T22:27:32.915455263Z","region_metadata_checked_at":"2026-09-21T22:27:32.915455263Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"973727765b758fc1d3f9380162839a4815e36063675e6cb800e5ee4c3fa948af","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102125829403185152","occurred_at":"2026-09-21T20:00:27Z","failed_at":"e2e","signature_id":"a861414d1e57f127bb5a66b1d8159aff733d41f5860f88297528b93c963c34dd","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001b82168\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-w27ffgnpjbb6\" to finish deleting: GET https://rp.j3185152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bcadd2fc-43d9-42e6-a28f-d1cbf161976e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bcadd2fc-43d9-42e6-a28f-d1cbf161976e\",\n      \"name\": \"bcadd2fc-43d9-42e6-a28f-d1cbf161976e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T21:36:07.059509255Z\",\n      \"endTime\": \"2026-09-21T22:00:10.226793643Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svkkujqqpiut9tmlj3qr0l60mhiko2m still exists (deletion dispatched at 2026-09-21T21:36:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001b82168\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-w27ffgnpjbb6\" to finish deleting: GET https://rp.j3185152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bcadd2fc-43d9-42e6-a28f-d1cbf161976e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bcadd2fc-43d9-42e6-a28f-d1cbf161976e\",\n      \"name\": \"bcadd2fc-43d9-42e6-a28f-d1cbf161976e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T21:36:07.059509255Z\",\n      \"endTime\": \"2026-09-21T22:00:10.226793643Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svkkujqqpiut9tmlj3qr0l60mhiko2m still exists (deletion dispatched at 2026-09-21T21:36:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102122240874123264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6945,"pr_state":"closed","pr_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","final_merged_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T19:46:12Z","started_at":"2026-09-21T19:46:12Z","completed_at":"2026-09-21T21:49:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T21:56:15.729650054Z","timing_metadata_checked_at":"2026-09-21T21:56:15.729650054Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T21:56:15.732474746Z","region_metadata_checked_at":"2026-09-21T21:56:15.732474746Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"05ee483919483684ccd613802fc7f2061cd7fb7c7c15a16c92778b8320487d0e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102122240874123264","occurred_at":"2026-09-21T19:46:12Z","failed_at":"e2e","signature_id":"1d53937632ebfdcc294d37645f71c88c8bbf3994286133bc420a9b0b4f84859c","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000da24c8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-jh2vpqrbl9c6\" to finish deleting: GET https://rp.j4123264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc4e1d95-f3cc-4562-9145-debc00ad20f3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc4e1d95-f3cc-4562-9145-debc00ad20f3\",\n      \"name\": \"cc4e1d95-f3cc-4562-9145-debc00ad20f3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T21:07:30.605613608Z\",\n      \"endTime\": \"2026-09-21T21:31:36.667443292Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svk6ogmv8h0au0imcrr7d3pbp9m040o still exists (deletion dispatched at 2026-09-21T21:07:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000da24c8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-jh2vpqrbl9c6\" to finish deleting: GET https://rp.j4123264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc4e1d95-f3cc-4562-9145-debc00ad20f3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc4e1d95-f3cc-4562-9145-debc00ad20f3\",\n      \"name\": \"cc4e1d95-f3cc-4562-9145-debc00ad20f3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T21:07:30.605613608Z\",\n      \"endTime\": \"2026-09-21T21:31:36.667443292Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svk6ogmv8h0au0imcrr7d3pbp9m040o still exists (deletion dispatched at 2026-09-21T21:07:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"25dac140c9810393db21fbef59e47d462a97f5bf2c28d808b1772a4f9a66aac7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102122240874123264","occurred_at":"2026-09-21T19:46:12Z","failed_at":"e2e","signature_id":"98eaec67c35f8ebf56c407f49f771122f6cf554a92aacc2b5555f3ffb37f37ee","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00116e3a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-d67dvdcr9ktk\" to finish deleting: GET https://rp.j4123264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0d831c64-0170-47ae-9e14-b054c5896d90\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0d831c64-0170-47ae-9e14-b054c5896d90\",\n      \"name\": \"0d831c64-0170-47ae-9e14-b054c5896d90\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T20:53:06.191785991Z\",\n      \"endTime\": \"2026-09-21T21:17:11.614187754Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svk7nstokpg5ko53a30cldfct8hi0jp still exists (deletion dispatched at 2026-09-21T20:53:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00116e3a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-d67dvdcr9ktk\" to finish deleting: GET https://rp.j4123264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0d831c64-0170-47ae-9e14-b054c5896d90\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0d831c64-0170-47ae-9e14-b054c5896d90\",\n      \"name\": \"0d831c64-0170-47ae-9e14-b054c5896d90\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T20:53:06.191785991Z\",\n      \"endTime\": \"2026-09-21T21:17:11.614187754Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svk7nstokpg5ko53a30cldfct8hi0jp still exists (deletion dispatched at 2026-09-21T20:53:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5a140845a2acac83a45cc998efe9239cf65ae648b6b685c73a33943ef6100f1c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102122240874123264","occurred_at":"2026-09-21T19:46:12Z","failed_at":"alert","signature_id":"3e285bc0fed84dddceb4aedbe9f05aa40edec5f29b85b33e5a8b6a3cdea0aee3","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-21T21:29:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j4123264-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-m4kdz\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-m4kdz (namespace arobit) on cluster ci01-j4123264-svc has averaged 127.7% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-21T21:30:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j4123264-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-58zl4\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-58zl4 (namespace arobit) on cluster ci01-j4123264-mgmt-2 has averaged 122% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"f2b6fbbb7dd76a4ea11b617f9109f139f2892d190a23244a6e9109f15b30be53","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102122240874123264","occurred_at":"2026-09-21T19:46:12Z","failed_at":"e2e","signature_id":"6dc6c1479f1127dca49f9265aaf53e60677a4f9526591e3435a015c85611c4ef","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00117e228\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-fcb9q2\" in resourcegroup=\"rg-np-version-upgrade-fcb9q2-5fvvn6dvbfw2\" to finish deleting: GET https://rp.j4123264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b73b986a-cbd8-46ad-a98a-47e23510dd86\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b73b986a-cbd8-46ad-a98a-47e23510dd86\",\n      \"name\": \"b73b986a-cbd8-46ad-a98a-47e23510dd86\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T21:03:18.285006192Z\",\n      \"endTime\": \"2026-09-21T21:27:26.64845298Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svk6sulcftrniu85kt4tediq8bh5ulc still exists (deletion dispatched at 2026-09-21T21:03:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00117e228\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-fcb9q2\" in resourcegroup=\"rg-np-version-upgrade-fcb9q2-5fvvn6dvbfw2\" to finish deleting: GET https://rp.j4123264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b73b986a-cbd8-46ad-a98a-47e23510dd86\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b73b986a-cbd8-46ad-a98a-47e23510dd86\",\n      \"name\": \"b73b986a-cbd8-46ad-a98a-47e23510dd86\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T21:03:18.285006192Z\",\n      \"endTime\": \"2026-09-21T21:27:26.64845298Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svk6sulcftrniu85kt4tediq8bh5ulc still exists (deletion dispatched at 2026-09-21T21:03:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102120305865527296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6925,"pr_state":"open","pr_sha":"9ef5982dac7a7b22811f15404810c31d0372579b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T19:38:31Z","started_at":"2026-09-21T19:38:31Z","completed_at":"2026-09-21T21:53:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T21:56:15.398672422Z","timing_metadata_checked_at":"2026-09-21T21:56:15.398672422Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T21:56:15.401303215Z","region_metadata_checked_at":"2026-09-21T21:56:15.401303215Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"10c3d810325d95adbe44e9b38b799b6f4c17aa2696ed7cd1fc41b4f854b7edff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102120305865527296","occurred_at":"2026-09-21T19:38:31Z","failed_at":"alert","signature_id":"5611ac078ddc9e5127435651a3844ee7f9599e86234c560542546354e7390c45","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T21:03:18Z\n  Ended:       2026-09-21T21:18:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5527296-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.33:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5527296/providers/microsoft.monitor/accounts/hcps-j5527296\", namespace=\"ocm-arohcpci01-2svk9tvinsjpvsqf4f95mmif44fb0pv7-v4j2g0l9y3n4g2w\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-qjq7d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2svk9tvinsjpvsqf4f95mmif44fb0pv7-v4j2g0l9y3n4g2w/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"12c9a1b56dbac08088bbd10c543b8ae213f13af674059a704374f47eadeea3b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102120305865527296","occurred_at":"2026-09-21T19:38:31Z","failed_at":"e2e","signature_id":"21fbf9f65c87d1d66ad8e51d13298b454806dc8fbfbd56cdafeb7a57d49c5326","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-d99twcmkwh84/cluster-candidate-5-0-8br7x9 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001010280\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-8br7x9: failed waiting for cluster=\"cluster-candidate-5-0-8br7x9\" in resourcegroup=\"rg-candidate-5-0-d99twcmkwh84\" to finish creating: GET https://rp.j5527296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0799c7ba-4b49-4034-8d8b-2071f01c8afd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0799c7ba-4b49-4034-8d8b-2071f01c8afd\",\n      \"name\": \"0799c7ba-4b49-4034-8d8b-2071f01c8afd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T20:26:31.347180566Z\",\n      \"endTime\": \"2026-09-21T20:55:38.332601158Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-d99twcmkwh84/cluster-candidate-5-0-8br7x9 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001010280\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-8br7x9: failed waiting for cluster=\"cluster-candidate-5-0-8br7x9\" in resourcegroup=\"rg-candidate-5-0-d99twcmkwh84\" to finish creating: GET https://rp.j5527296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0799c7ba-4b49-4034-8d8b-2071f01c8afd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0799c7ba-4b49-4034-8d8b-2071f01c8afd\",\n      \"name\": \"0799c7ba-4b49-4034-8d8b-2071f01c8afd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T20:26:31.347180566Z\",\n      \"endTime\": \"2026-09-21T20:55:38.332601158Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1461847238bb176ac579ff9c6f653a15aac525c361d7334dff02e974f90b7ff6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102120305865527296","occurred_at":"2026-09-21T19:38:31Z","failed_at":"alert","signature_id":"8754c0dcc2004a1dc3d9e3c7de0ac4bba1bae5af98c84c85241dde6028adfc87","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T20:49:23Z\n  Ended:       2026-09-21T21:18:25Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j5527296-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.33:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5527296/providers/microsoft.monitor/accounts/hcps-j5527296\", namespace=\"ocm-arohcpci01-2svk9tvinsjpvsqf4f95mmif44fb0pv7-v4j2g0l9y3n4g2w\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-qjq7d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2svk9tvinsjpvsqf4f95mmif44fb0pv7-v4j2g0l9y3n4g2w/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-21T21:00:23Z\n  Ended:       2026-09-21T21:14:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j5527296-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.33:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5527296/providers/microsoft.monitor/accounts/hcps-j5527296\", namespace=\"ocm-arohcpci01-2svk9tvinsjpvsqf4f95mmif44fb0pv7-v4j2g0l9y3n4g2w\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-qjq7d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2svk9tvinsjpvsqf4f95mmif44fb0pv7-v4j2g0l9y3n4g2w/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"b3614ebf6010cc39072be758e5c8aac53e5af04376e68a4cd1e0de6018128c42","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102120305865527296","occurred_at":"2026-09-21T19:38:31Z","failed_at":"e2e","signature_id":"933d6687811952f3fec32125cf0b0afe14bcf8dd8763c01e3d349f67ee158f9b","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001ad20f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-xzkzgr7f4c7m\" to finish deleting: GET https://rp.j5527296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/abee6f31-d0fb-4c57-bba6-2a10f1062250\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/abee6f31-d0fb-4c57-bba6-2a10f1062250\",\n      \"name\": \"abee6f31-d0fb-4c57-bba6-2a10f1062250\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T21:12:09.871466085Z\",\n      \"endTime\": \"2026-09-21T21:36:16.735162069Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svka2v4pd8vc8gfas9ehv6snsojtc2p still exists (deletion dispatched at 2026-09-21T21:12:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001ad20f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-xzkzgr7f4c7m\" to finish deleting: GET https://rp.j5527296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/abee6f31-d0fb-4c57-bba6-2a10f1062250\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/abee6f31-d0fb-4c57-bba6-2a10f1062250\",\n      \"name\": \"abee6f31-d0fb-4c57-bba6-2a10f1062250\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T21:12:09.871466085Z\",\n      \"endTime\": \"2026-09-21T21:36:16.735162069Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svka2v4pd8vc8gfas9ehv6snsojtc2p still exists (deletion dispatched at 2026-09-21T21:12:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102116740375777280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5846,"pr_state":"closed","pr_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","final_merged_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T19:24:20Z","started_at":"2026-09-21T19:24:20Z","completed_at":"2026-09-21T21:25:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T21:45:19.621053027Z","timing_metadata_checked_at":"2026-09-21T21:45:19.621053027Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T21:45:19.625106114Z","region_metadata_checked_at":"2026-09-21T21:45:19.625106114Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"7faebd083dbcdb0e95a0a026d7b1bceb0f64fa96407c852b4899b905c24e0c1d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102116740375777280","occurred_at":"2026-09-21T19:24:20Z","failed_at":"alert","signature_id":"8d742ce5ed191fa92313bd72fc22c26ffe7de709e375bb13b6e80081923bd689","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T20:17:35Z\n  Ended:       2026-09-21T20:50:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j5777280-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j5777280-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102116189634301952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7020,"pr_state":"open","pr_sha":"6cb0492059ca17ab9fe7fffef3fab0908d31e190","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T19:22:09Z","started_at":"2026-09-21T19:22:09Z","completed_at":"2026-09-21T21:44:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T21:56:15.888914336Z","timing_metadata_checked_at":"2026-09-21T21:56:15.888914336Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T21:56:15.892150127Z","region_metadata_checked_at":"2026-09-21T21:56:15.892150127Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"ad0da03a279f276a2ffb2ecb31103e438e8da42383144165050c6256d533ef54","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102116189634301952","occurred_at":"2026-09-21T19:22:09Z","failed_at":"e2e","signature_id":"abd83a72f84835b7ba0e22c9458792d9debb3af01b78cf0a7a35bc08262d161a","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00199e138\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-tlnl48rwzl28\" to finish deleting: GET https://rp.j4301952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17d14834-4d3c-4230-a071-aa42d852b8bb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17d14834-4d3c-4230-a071-aa42d852b8bb\",\n      \"name\": \"17d14834-4d3c-4230-a071-aa42d852b8bb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T21:00:09.200038175Z\",\n      \"endTime\": \"2026-09-21T21:24:12.870028927Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svk3vpqptq2l3v57g6h4msro6gkj7t1 still exists (deletion dispatched at 2026-09-21T21:00:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00199e138\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-tlnl48rwzl28\" to finish deleting: GET https://rp.j4301952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17d14834-4d3c-4230-a071-aa42d852b8bb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17d14834-4d3c-4230-a071-aa42d852b8bb\",\n      \"name\": \"17d14834-4d3c-4230-a071-aa42d852b8bb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T21:00:09.200038175Z\",\n      \"endTime\": \"2026-09-21T21:24:12.870028927Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svk3vpqptq2l3v57g6h4msro6gkj7t1 still exists (deletion dispatched at 2026-09-21T21:00:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102114596662808576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7054,"pr_state":"open","pr_sha":"ee75ffdcc6c77d0c69fd094fae602aaa8f0565ea","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T19:15:49Z","started_at":"2026-09-21T19:15:49Z","completed_at":"2026-09-21T21:14:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T21:25:01.742905411Z","timing_metadata_checked_at":"2026-09-21T21:25:01.742905411Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T21:25:01.745799402Z","region_metadata_checked_at":"2026-09-21T21:25:01.745799402Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"3ad8af5aa57c5a1d989e02fb07a7bc1b5bb8b0dc171921578804fcdee7a2fee7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102114596662808576","occurred_at":"2026-09-21T19:15:49Z","failed_at":"e2e","signature_id":"7b09f0e5d5b4ec2dbf734415baedb473fa24df0e8fd83c12cc61c5a8f083dfb3","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012ee3a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-n5cb426wnppn\" to finish deleting: GET https://rp.j2808576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8d44152-0978-4486-87f4-1c2e97aa9260\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8d44152-0978-4486-87f4-1c2e97aa9260\",\n      \"name\": \"c8d44152-0978-4486-87f4-1c2e97aa9260\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T20:32:01.060772851Z\",\n      \"endTime\": \"2026-09-21T20:56:10.462817257Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svjsbu7q89jafjlndsvrctgqkh4j16a still exists (deletion dispatched at 2026-09-21T20:32:03Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012ee3a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-n5cb426wnppn\" to finish deleting: GET https://rp.j2808576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8d44152-0978-4486-87f4-1c2e97aa9260\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8d44152-0978-4486-87f4-1c2e97aa9260\",\n      \"name\": \"c8d44152-0978-4486-87f4-1c2e97aa9260\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T20:32:01.060772851Z\",\n      \"endTime\": \"2026-09-21T20:56:10.462817257Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svjsbu7q89jafjlndsvrctgqkh4j16a still exists (deletion dispatched at 2026-09-21T20:32:03Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e5b7b3b38ff6104b1b815afbbec1997d750c06ce5a01f585f8c89989fdd09d32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102114596662808576","occurred_at":"2026-09-21T19:15:49Z","failed_at":"alert","signature_id":"384d2d5dadf32e155c9c6d56cb153b73edc7f3291263d3ba02d742eb0041e689","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T20:05:03Z\n  Ended:       2026-09-21T20:39:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j2808576-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j2808576-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102108631234252800","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6945,"pr_state":"closed","pr_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","final_merged_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T18:52:07Z","started_at":"2026-09-21T18:52:07Z","completed_at":"2026-09-21T19:35:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T19:40:39.251122563Z","timing_metadata_checked_at":"2026-09-21T19:40:39.251122563Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T19:40:39.254229255Z","region_metadata_checked_at":"2026-09-21T19:40:39.254229255Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"902f92af6ed081a96e3fa4449d3e051ef700c182001da8be2fd5da00cddccd7c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102108631234252800","occurred_at":"2026-09-21T18:52:07Z","failed_at":"provision","signature_id":"920f3c74c140ea63d3fe5bed0bd9d83eedd253f0efd83b157445d28cdec20771","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/swift-vnet-permissions (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-21T19:22:42.604Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet-permissions stamp=2 description=\"Step swift-vnet-permissions\\n  Kind: ARM\\n  Template: templates/swift-vnet-permissions.bicep\\n  Parameters: configurations/swift-vnet-permissions.tmpl.bicepparam\"\ntime=2026-09-21T19:23:35.587Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet-permissions stamp=2\ntime=2026-09-21T19:25:31.977Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet-permissions stamp=2 err=\"stamp 2: failed to run ARM step: failed to create deployment: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4252800-mgmt-2/providers/Microsoft.Resources/deployments/681f37eea538402bd4d4d12ef523e5767a0f365997d5c7810d0e142ad8086026\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServerTimeout\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"ServerTimeout\\\",\\n    \\\"message\\\": \\\"The request timed out. Diagnostic information: timestamp '20260921T192531Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '0d759d1e-a59a-434f-931d-09ba48ef8f44', request correlation id '0d759d1e-a59a-434f-931d-09ba48ef8f44'.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e41c14d827a6393f322e3a16884e1f60b22ca58ea533fcbde300b78420566e85","failure_pattern":"ERROR CODE: ServerTimeout","search_query":"ERROR CODE: ServerTimeout"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e41c14d827a6393f322e3a16884e1f60b22ca58ea533fcbde300b78420566e85"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7055/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102100127102537728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7055,"pr_state":"open","pr_sha":"7ded4c0b4d198489f37f011b88bc6dd20ce27163","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T18:18:20Z","started_at":"2026-09-21T18:18:20Z","completed_at":"2026-09-21T18:39:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T18:47:47.9607353Z","timing_metadata_checked_at":"2026-09-21T18:47:47.9607353Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T18:47:47.964361388Z","region_metadata_checked_at":"2026-09-21T18:47:47.964361388Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"25286e255af2c904794000b961ae0ddeee6ec3b77f82138a4c7e27cdaa485536","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7055/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102100127102537728","occurred_at":"2026-09-21T18:18:20Z","failed_at":"provision","signature_id":"fdb3fc6bea530a113f7b0ee9cc1efb974065f796a13284f1a617ab68739d648a","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/admin-api-cert","test_suite":"templatize-pipeline","failure_text":"time=2026-09-21T18:34:41.812Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert description=\"Step admin-api-cert\\n Kind: CreateCertificate\\n\"\ntime=2026-09-21T18:34:41.812Z level=INFO msg=\"Creating certificate\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert vaultBaseUrl=https://aro-hcp-dev-svc-kv.vault.azure.net/ certificateName=admin-api-cert-ci01-j2537728 issuer=Self commonName=admin.j2537728.hcpsvc.osadev.cloud san=admin.j2537728.hcpsvc.osadev.cloud\ntime=2026-09-21T18:34:42.532Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert err=\"error running Create Certificate Step: failed to create certificate \\\"admin-api-cert-ci01-j2537728\\\" in vault \\\"https://aro-hcp-dev-svc-kv.vault.azure.net/\\\": POST https://aro-hcp-dev-svc-kv.vault.azure.net/certificates/admin-api-cert-ci01-j2537728/create\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Certificate admin-api-cert-ci01-j2537728 is currently in a deleted but recoverable state, and its name cannot be reused; in this state, the certificate can only be recovered or purged.\\\",\\n    \\\"innererror\\\": {\\n      \\\"code\\\": \\\"ObjectIsDeletedButRecoverable\\\"\\n    }\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6f6e7e66ad8bc42bd6647f1e246b1e77205441e0c9495974d9b1f940d8bdec9c","failure_pattern":"ERROR CODE: Conflict; detail code ObjectIsDeletedButRecoverable","search_query":"ERROR CODE: Conflict"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6f6e7e66ad8bc42bd6647f1e246b1e77205441e0c9495974d9b1f940d8bdec9c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102098571925917696","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7053,"pr_state":"open","pr_sha":"53b89aaf4bed03994f369439935a28a911290c9d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T18:12:09Z","started_at":"2026-09-21T18:12:09Z","completed_at":"2026-09-21T18:48:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T18:58:53.01389665Z","timing_metadata_checked_at":"2026-09-21T18:58:53.01389665Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T18:58:53.016593842Z","region_metadata_checked_at":"2026-09-21T18:58:53.016593842Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"a135ae69b3192f41140491d9910d8b178848702cdbaaede9d22c7afafed920be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102098571925917696","occurred_at":"2026-09-21T18:12:09Z","failed_at":"provision","signature_id":"85d04ded75c01f0de4d7f4f9135c9d77acc8ba7a2fbbb0984961236c0d3c0b07","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-21T18:34:08.196Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-09-21T18:41:45.259Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-21T18:34:08.202514223Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5917696-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5917696-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\"}\\n{\\\"time\\\":\\\"2026-09-21T18:34:09.977044373Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5917696-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5917696-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-21T18:34:10.113714041Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5917696-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5917696-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-21T18:41:45.256724505Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5917696-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5917696-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"63BD079BDF59F7AF\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5917696-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5917696-mgmt-1/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j5917696-mgmt-1/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-21T18:34:14.3081847Z\\\",\\\"duration\\\":\\\"PT7M21.1354492S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-21T18:41:45.256785637Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5917696-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5917696-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"66DC8ACA57717BD0\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5917696-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5917696-mgmt-1/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j5917696-mgmt-1/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-21T18:39:37.4650836Z\\\",\\\"duration\\\":\\\"PT4.3067939S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 6 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-21T18:41:45.256818608Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5917696-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5917696-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"A4AD211D64497EC2\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5917696-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5917696-mgmt-1/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j5917696-mgmt-1/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-21T18:39:37.4650836Z\\\",\\\"duration\\\":\\\"PT1M56.945679S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-21T18:41:45.256835428Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5917696-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5917696-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"FAD9AF8022EC4353\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5917696-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5917696-mgmt-1/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j5917696-mgmt-1/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-21T18:39:37.4650836Z\\\",\\\"duration\\\":\\\"PT3.9630448S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 6 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-21T18:41:45.256866349Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5917696-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5917696-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"E51E80C79F3CC641\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5917696-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5917696-mgmt-1/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j5917696-mgmt-1/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-21T18:39:37.4650836Z\\\",\\\"duration\\\":\\\"PT3.4005374S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 6 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-21T18:41:45.25689421Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5917696-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5917696-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"640F2A04A52AF91E\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5917696-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5917696-mgmt-1\\\",\\\"resourceName\\\":\\\"ci01-j5917696-mgmt-1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-21T18:34:14.3081847Z\\\",\\\"duration\\\":\\\"PT5M23.0886527S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-21T18:41:45.25691446Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5917696-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5917696-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5917696-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584115912350974327\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5917696-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n    \\\\\\\"details\\\\\\\": [\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5917696-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5917696-mgmt-1/agentPools/userswft3\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 6 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n      },\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5917696-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5917696-mgmt-1/agentPools/userswft1\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 6 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n      },\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5917696-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5917696-mgmt-1/agentPools/userswft2\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 6 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n      }\\\\n    ]\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"7m37.054409927s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cd0b0b8f7ac9ea6402ca4eac6d805c330783a014b8096e0958d5137c2c282bdb","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ErrCode_InsufficientVCPUQuota; detail message Insufficient vcpu quota requested \u003ccount\u003e, remaining \u003ccount\u003e for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: \u003curl\u003e Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["cd0b0b8f7ac9ea6402ca4eac6d805c330783a014b8096e0958d5137c2c282bdb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102097561530667008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7020,"pr_state":"open","pr_sha":"6cb0492059ca17ab9fe7fffef3fab0908d31e190","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T18:08:08Z","started_at":"2026-09-21T18:08:08Z","completed_at":"2026-09-21T18:31:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T18:37:06.274286088Z","timing_metadata_checked_at":"2026-09-21T18:37:06.274286088Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T18:37:06.277554277Z","region_metadata_checked_at":"2026-09-21T18:37:06.277554277Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"07370adc224ed05ee711335415a4a5995c405bb259f180d21b24f8f6dc8bf2c2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102097561530667008","occurred_at":"2026-09-21T18:08:08Z","failed_at":"provision","signature_id":"fb61014e0da0dadc04a3117634aef1ed0d31164a421ed411c631d509dfe950f0","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/admin-api-cert","test_suite":"templatize-pipeline","failure_text":"time=2026-09-21T18:25:15.815Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert description=\"Step admin-api-cert\\n Kind: CreateCertificate\\n\"\ntime=2026-09-21T18:25:15.815Z level=INFO msg=\"Creating certificate\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert vaultBaseUrl=https://aro-hcp-dev-svc-kv.vault.azure.net/ certificateName=admin-api-cert-ci01-j0667008 issuer=Self commonName=admin.j0667008.hcpsvc.osadev.cloud san=admin.j0667008.hcpsvc.osadev.cloud\ntime=2026-09-21T18:25:16.432Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert err=\"error running Create Certificate Step: failed to create certificate \\\"admin-api-cert-ci01-j0667008\\\" in vault \\\"https://aro-hcp-dev-svc-kv.vault.azure.net/\\\": POST https://aro-hcp-dev-svc-kv.vault.azure.net/certificates/admin-api-cert-ci01-j0667008/create\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Certificate admin-api-cert-ci01-j0667008 is currently in a deleted but recoverable state, and its name cannot be reused; in this state, the certificate can only be recovered or purged.\\\",\\n    \\\"innererror\\\": {\\n      \\\"code\\\": \\\"ObjectIsDeletedButRecoverable\\\"\\n    }\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6f6e7e66ad8bc42bd6647f1e246b1e77205441e0c9495974d9b1f940d8bdec9c","failure_pattern":"ERROR CODE: Conflict; detail code ObjectIsDeletedButRecoverable","search_query":"ERROR CODE: Conflict"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6f6e7e66ad8bc42bd6647f1e246b1e77205441e0c9495974d9b1f940d8bdec9c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102096991210180608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5562,"pr_state":"closed","pr_sha":"8238b6d861eb15be629bec015ec651ec46956af8","final_merged_sha":"18cce3707ed7df30b1a12d9779a8035dbdc24710","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T18:05:52Z","started_at":"2026-09-21T18:05:52Z","completed_at":"2026-09-21T18:49:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T18:58:50.09906508Z","timing_metadata_checked_at":"2026-09-21T18:58:50.09906508Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T18:58:50.105046362Z","region_metadata_checked_at":"2026-09-21T18:58:50.105046362Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"25bad6c2dd567c50baf4646ec0fa280e60a002d774cff27d47f41d8e09b08f86","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102096991210180608","occurred_at":"2026-09-21T18:05:52Z","failed_at":"provision","signature_id":"d88409b59c56092375cca67e126827f50822709f7340f7ccdcf466f7b093ade6","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-21T18:33:10.189Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-09-21T18:43:18.824Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-21T18:33:10.196040039Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0180608-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j0180608-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\"}\\n{\\\"time\\\":\\\"2026-09-21T18:33:12.013713201Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0180608-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j0180608-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-21T18:33:12.155143709Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0180608-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j0180608-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-21T18:43:18.82214017Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0180608-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j0180608-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"2B38A36D9799EB2E\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0180608-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j0180608-mgmt-1/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j0180608-mgmt-1/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-21T18:33:16.7353105Z\\\",\\\"duration\\\":\\\"PT9M44.0435529S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-21T18:43:18.822198134Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0180608-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j0180608-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"D52FC66163407372\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0180608-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j0180608-mgmt-1/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j0180608-mgmt-1/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-21T18:41:21.3624777Z\\\",\\\"duration\\\":\\\"PT1M41.8443951S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-21T18:43:18.822211544Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0180608-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j0180608-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"8650A5AE123230FA\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0180608-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j0180608-mgmt-1/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j0180608-mgmt-1/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-21T18:41:21.3624777Z\\\",\\\"duration\\\":\\\"PT4.1420203S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 6 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-21T18:43:18.822237446Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0180608-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j0180608-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"7AC95EDE577F9BF6\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0180608-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j0180608-mgmt-1/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j0180608-mgmt-1/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-21T18:41:21.3624777Z\\\",\\\"duration\\\":\\\"PT4.0795266S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 6 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-21T18:43:18.822259057Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0180608-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j0180608-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"7610750267384434\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0180608-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j0180608-mgmt-1/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j0180608-mgmt-1/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-21T18:41:21.3624777Z\\\",\\\"duration\\\":\\\"PT3.4857707S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 6 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-21T18:43:18.822279099Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0180608-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j0180608-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"CCE749298B22E92C\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0180608-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j0180608-mgmt-1\\\",\\\"resourceName\\\":\\\"ci01-j0180608-mgmt-1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-21T18:33:16.7353105Z\\\",\\\"duration\\\":\\\"PT8M4.5504301S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-21T18:43:18.82230049Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0180608-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j0180608-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0180608-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584115912930590660\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0180608-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n    \\\\\\\"details\\\\\\\": [\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0180608-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j0180608-mgmt-1/agentPools/userswft1\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 6 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n      },\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0180608-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j0180608-mgmt-1/agentPools/userswft3\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 6 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n      },\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0180608-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j0180608-mgmt-1/agentPools/userswft2\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 6 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n      }\\\\n    ]\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"10m8.626268442s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cd0b0b8f7ac9ea6402ca4eac6d805c330783a014b8096e0958d5137c2c282bdb","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ErrCode_InsufficientVCPUQuota; detail message Insufficient vcpu quota requested \u003ccount\u003e, remaining \u003ccount\u003e for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: \u003curl\u003e Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["cd0b0b8f7ac9ea6402ca4eac6d805c330783a014b8096e0958d5137c2c282bdb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102095923193581568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7044,"pr_state":"closed","pr_sha":"0f625957a86b5fda4999ce65087e506bc2499c89","final_merged_sha":"0f625957a86b5fda4999ce65087e506bc2499c89","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T18:01:30Z","started_at":"2026-09-21T18:01:30Z","completed_at":"2026-09-21T20:20:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T20:22:06.402134012Z","timing_metadata_checked_at":"2026-09-21T20:22:06.402134012Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T20:22:06.405009703Z","region_metadata_checked_at":"2026-09-21T20:22:06.405009703Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"0fc77bb645726440096010fa87f62f618a553fd8b454805380edf63606c424c8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102095923193581568","occurred_at":"2026-09-21T18:01:30Z","failed_at":"e2e","signature_id":"84632ad89d72e4a347c9d226f46770e8a6b54bacd60804b4ec5117388add71c3","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0013dc198\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-tkfxb96x69sv\" to finish deleting: GET https://rp.j3581568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3b00304e-2ecc-46ca-ae98-3ccddf864d89\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3b00304e-2ecc-46ca-ae98-3ccddf864d89\",\n      \"name\": \"3b00304e-2ecc-46ca-ae98-3ccddf864d89\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T19:31:24.492768923Z\",\n      \"endTime\": \"2026-09-21T19:55:29.969595473Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svis2tla8egic6l75j1qsgmldgs1ilo still exists (deletion dispatched at 2026-09-21T19:31:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0013dc198\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-tkfxb96x69sv\" to finish deleting: GET https://rp.j3581568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3b00304e-2ecc-46ca-ae98-3ccddf864d89\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3b00304e-2ecc-46ca-ae98-3ccddf864d89\",\n      \"name\": \"3b00304e-2ecc-46ca-ae98-3ccddf864d89\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T19:31:24.492768923Z\",\n      \"endTime\": \"2026-09-21T19:55:29.969595473Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svis2tla8egic6l75j1qsgmldgs1ilo still exists (deletion dispatched at 2026-09-21T19:31:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5892624f3c7360176168a32aa49324366399f91ec590c0c1253cbfc582c2631d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102095923193581568","occurred_at":"2026-09-21T18:01:30Z","failed_at":"e2e","signature_id":"0b784dd0db2851a6c6950e6f32adeb8defbd766fa7108912932ac30ba58a966f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-fdh6f5\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fe4500\u003e: \n    DNS for route host agnhost-e2e-sample-app-c6klm.apps.aro.z4z2f7u9q3y4c5l.96pg.j3581568.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-c6klm.apps.aro.z4z2f7u9q3y4c5l.96pg.j3581568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-c6klm.apps.aro.z4z2f7u9q3y4c5l.96pg.j3581568.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-fdh6f5\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fe4500\u003e: \n    DNS for route host agnhost-e2e-sample-app-c6klm.apps.aro.z4z2f7u9q3y4c5l.96pg.j3581568.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-c6klm.apps.aro.z4z2f7u9q3y4c5l.96pg.j3581568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-c6klm.apps.aro.z4z2f7u9q3y4c5l.96pg.j3581568.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-28d65.apps.aro.o4p4d0n2f5l6i5t.cztj.j0999424.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-28d65.apps.aro.o4p4d0n2f5l6i5t.cztj.j0999424.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-28d65.apps.aro.o4p4d0n2f5l6i5t.cztj.j0999424.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"f4eb71cf0e36d4b973e67f8295889ae004606a97cb764796122ffc980a2ac8a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102095923193581568","occurred_at":"2026-09-21T18:01:30Z","failed_at":"e2e","signature_id":"d4437ad1918d253a95c2954a03bad7dcdbc697b3fc861162532f993e44063118","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014fa0c0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-44glsr6w8q8z\" to finish deleting: GET https://rp.j3581568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2681fdf1-9cee-4b43-aee8-7f56a104a4e9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2681fdf1-9cee-4b43-aee8-7f56a104a4e9\",\n      \"name\": \"2681fdf1-9cee-4b43-aee8-7f56a104a4e9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T19:37:47.04023198Z\",\n      \"endTime\": \"2026-09-21T20:01:49.985447236Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svis85uq6nb15p69p11455abjrv6hjt still exists (deletion dispatched at 2026-09-21T19:37:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014fa0c0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-44glsr6w8q8z\" to finish deleting: GET https://rp.j3581568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2681fdf1-9cee-4b43-aee8-7f56a104a4e9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2681fdf1-9cee-4b43-aee8-7f56a104a4e9\",\n      \"name\": \"2681fdf1-9cee-4b43-aee8-7f56a104a4e9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T19:37:47.04023198Z\",\n      \"endTime\": \"2026-09-21T20:01:49.985447236Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svis85uq6nb15p69p11455abjrv6hjt still exists (deletion dispatched at 2026-09-21T19:37:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102095920672804864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7033,"pr_state":"closed","pr_sha":"c448a0db20d77ea047c00534044b12cc2b2ac565","final_merged_sha":"ef821496d20065a3db001282a41d83827d994051","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T18:01:28Z","started_at":"2026-09-21T18:01:28Z","completed_at":"2026-09-21T20:26:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T20:32:31.734164861Z","timing_metadata_checked_at":"2026-09-21T20:32:31.734164861Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T20:32:31.737281652Z","region_metadata_checked_at":"2026-09-21T20:32:31.737281652Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"53a545d844815e689dd678270d17cbf08962df2d6ebf03b0ce715d98b469d6fd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102095920672804864","occurred_at":"2026-09-21T18:01:28Z","failed_at":"e2e","signature_id":"dc62e16f2d054eb6ecb3a730452e8258fe1deeb31b0e748a2e1ede8f7b0f87af","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:251]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0xc002a16b28\u003e: \n    context deadline exceeded\n    failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:251]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0xc002a16b28\u003e: \n    context deadline exceeded\n    failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"fe20baf27270ff77a587633ab67681a3a4d752ddf96eee1b1854b5f118799550","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102095920672804864","occurred_at":"2026-09-21T18:01:28Z","failed_at":"e2e","signature_id":"af4e73a608fe40ce6a9d85e6fffff041ced5ed3202e85cf33f00128b1e3874f6","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a64108\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-rnz7dwp5f767\" to finish deleting: GET https://rp.j2804864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0cc06aed-a13f-4124-924a-416f9bc7f92e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0cc06aed-a13f-4124-924a-416f9bc7f92e\",\n      \"name\": \"0cc06aed-a13f-4124-924a-416f9bc7f92e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T19:44:49.230317942Z\",\n      \"endTime\": \"2026-09-21T20:08:52.522049767Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svj0mggu7oacnc0vthkjuhpvaj4bumr still exists (deletion dispatched at 2026-09-21T19:44:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a64108\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-rnz7dwp5f767\" to finish deleting: GET https://rp.j2804864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0cc06aed-a13f-4124-924a-416f9bc7f92e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0cc06aed-a13f-4124-924a-416f9bc7f92e\",\n      \"name\": \"0cc06aed-a13f-4124-924a-416f9bc7f92e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T19:44:49.230317942Z\",\n      \"endTime\": \"2026-09-21T20:08:52.522049767Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svj0mggu7oacnc0vthkjuhpvaj4bumr still exists (deletion dispatched at 2026-09-21T19:44:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102095918189776896","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6995,"pr_state":"closed","pr_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","final_merged_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T18:01:22Z","started_at":"2026-09-21T18:01:22Z","completed_at":"2026-09-21T18:52:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T18:58:52.768590876Z","timing_metadata_checked_at":"2026-09-21T18:58:52.768590876Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T18:58:52.771814466Z","region_metadata_checked_at":"2026-09-21T18:58:52.771814466Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"84a1747d46e0ed11d21f883b2b1736c8a630a4183b958a0460ac045e23d9a2cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102095918189776896","occurred_at":"2026-09-21T18:01:22Z","failed_at":"provision","signature_id":"a3aa5c46f79378311590646156822d985ff2337a409a7614316817c870c6d4bd","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-21T18:32:22.796Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-09-21T18:45:32.382Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-21T18:32:22.803573824Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j9776896-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j9776896-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\"}\\n{\\\"time\\\":\\\"2026-09-21T18:32:24.805955581Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j9776896-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j9776896-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-21T18:32:24.940582858Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j9776896-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j9776896-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-21T18:45:32.379977629Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j9776896-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j9776896-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"EBBBD7B642ED7B1C\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9776896-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j9776896-mgmt-1/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j9776896-mgmt-1/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-21T18:43:20.9144849Z\\\",\\\"duration\\\":\\\"PT4.4919188S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 6 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-21T18:45:32.380066075Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j9776896-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j9776896-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"379E505D806B0369\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9776896-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j9776896-mgmt-1/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j9776896-mgmt-1/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-21T18:32:30.2828584Z\\\",\\\"duration\\\":\\\"PT12M29.3745304S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-21T18:45:32.380090146Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j9776896-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j9776896-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"5262EBE64FE1B417\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9776896-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j9776896-mgmt-1/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j9776896-mgmt-1/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-21T18:43:20.9144849Z\\\",\\\"duration\\\":\\\"PT3.9450427S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 6 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-21T18:45:32.380121888Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j9776896-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j9776896-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"ED59D84AC1FC6A35\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9776896-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j9776896-mgmt-1/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j9776896-mgmt-1/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-21T18:43:20.9144849Z\\\",\\\"duration\\\":\\\"PT3.7887958S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 6 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-21T18:45:32.38015029Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j9776896-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j9776896-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"4C2369403F5BC6DA\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9776896-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j9776896-mgmt-1/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j9776896-mgmt-1/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-21T18:43:20.9144849Z\\\",\\\"duration\\\":\\\"PT1M43.0824083S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-21T18:45:32.3801626Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j9776896-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j9776896-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"029C5991DAA3D2F1\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9776896-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j9776896-mgmt-1\\\",\\\"resourceName\\\":\\\"ci01-j9776896-mgmt-1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-21T18:32:30.2828584Z\\\",\\\"duration\\\":\\\"PT10M50.5766775S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-21T18:45:32.380190652Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j9776896-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j9776896-mgmt-1\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9776896-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584115913402802699\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9776896-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n    \\\\\\\"details\\\\\\\": [\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9776896-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j9776896-mgmt-1/agentPools/userswft3\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 6 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n      },\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9776896-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j9776896-mgmt-1/agentPools/userswft2\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 6 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n      },\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9776896-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j9776896-mgmt-1/agentPools/userswft1\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 6 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n      }\\\\n    ]\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"13m9.576625468s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cd0b0b8f7ac9ea6402ca4eac6d805c330783a014b8096e0958d5137c2c282bdb","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ErrCode_InsufficientVCPUQuota; detail message Insufficient vcpu quota requested \u003ccount\u003e, remaining \u003ccount\u003e for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: \u003curl\u003e Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["cd0b0b8f7ac9ea6402ca4eac6d805c330783a014b8096e0958d5137c2c282bdb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102095915643834368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6886,"pr_state":"closed","pr_sha":"a6aa6a03a137ec55e3f4f5d06cc620e6bf6dd7f3","final_merged_sha":"a6aa6a03a137ec55e3f4f5d06cc620e6bf6dd7f3","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T18:01:20Z","started_at":"2026-09-21T18:01:20Z","completed_at":"2026-09-21T20:21:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T20:32:31.494307151Z","timing_metadata_checked_at":"2026-09-21T20:32:31.494307151Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T20:32:31.498565138Z","region_metadata_checked_at":"2026-09-21T20:32:31.498565138Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"11fb966b14783c98f88f49fc41695d5938d5bc7b993908ca12a1604794eed67d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102095915643834368","occurred_at":"2026-09-21T18:01:20Z","failed_at":"e2e","signature_id":"38014781d11e14f4ab61a54cb8e2ceaf057f4c971cd02f001d6a59b001c6474e","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001080078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-b2tvxk5mglhg\" to finish deleting: GET https://rp.j3834368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6cc9f7df-85d3-4767-985c-ff24e87da875\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6cc9f7df-85d3-4767-985c-ff24e87da875\",\n      \"name\": \"6cc9f7df-85d3-4767-985c-ff24e87da875\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T19:21:01.145382289Z\",\n      \"endTime\": \"2026-09-21T19:45:07.883680578Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svits9kg40fldp0un40h3tt4rbi8phm still exists (deletion dispatched at 2026-09-21T19:21:02Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001080078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-b2tvxk5mglhg\" to finish deleting: GET https://rp.j3834368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6cc9f7df-85d3-4767-985c-ff24e87da875\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6cc9f7df-85d3-4767-985c-ff24e87da875\",\n      \"name\": \"6cc9f7df-85d3-4767-985c-ff24e87da875\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T19:21:01.145382289Z\",\n      \"endTime\": \"2026-09-21T19:45:07.883680578Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svits9kg40fldp0un40h3tt4rbi8phm still exists (deletion dispatched at 2026-09-21T19:21:02Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"441f6a2ff14d87e72f6be701bdfdd18474712cfa1cc859e3f69978f54b3d92c5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102095915643834368","occurred_at":"2026-09-21T18:01:20Z","failed_at":"e2e","signature_id":"7d89cdee31aea1b9d00181d528c3aebeb4f6c0f24af9675d98ca3b220eb818f9","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010dc168\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-qcrmvbvmz6kc\" to finish deleting: GET https://rp.j3834368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/64818a40-9262-4c39-b8a2-f4797497a78d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/64818a40-9262-4c39-b8a2-f4797497a78d\",\n      \"name\": \"64818a40-9262-4c39-b8a2-f4797497a78d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T19:38:43.097033808Z\",\n      \"endTime\": \"2026-09-21T20:02:44.67317846Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sviu7h5hscbssbbfu5plfq0ggiga766 still exists (deletion dispatched at 2026-09-21T19:38:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010dc168\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-qcrmvbvmz6kc\" to finish deleting: GET https://rp.j3834368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/64818a40-9262-4c39-b8a2-f4797497a78d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/64818a40-9262-4c39-b8a2-f4797497a78d\",\n      \"name\": \"64818a40-9262-4c39-b8a2-f4797497a78d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T19:38:43.097033808Z\",\n      \"endTime\": \"2026-09-21T20:02:44.67317846Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sviu7h5hscbssbbfu5plfq0ggiga766 still exists (deletion dispatched at 2026-09-21T19:38:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4dda3132b4ec70cd3b33116a2d869f870fbe6b4028c3ca9a71731479f8d6c441","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102095915643834368","occurred_at":"2026-09-21T18:01:20Z","failed_at":"e2e","signature_id":"e0ca0648fc3e56acb4177ea709f1b139941a8d44c165a28d546a2c38dd4ea096","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: failed to wait for second cluster \"basic-hcp-cluster2\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00151cd50\u003e: \n    GET https://rp.j3834368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8dbab85-98a4-49d4-94ad-b689a0da487b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8dbab85-98a4-49d4-94ad-b689a0da487b\",\n      \"name\": \"e8dbab85-98a4-49d4-94ad-b689a0da487b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T18:52:59.240619957Z\",\n      \"endTime\": \"2026-09-21T19:22:07.757736289Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: failed to wait for second cluster \"basic-hcp-cluster2\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00151cd50\u003e: \n    GET https://rp.j3834368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8dbab85-98a4-49d4-94ad-b689a0da487b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8dbab85-98a4-49d4-94ad-b689a0da487b\",\n      \"name\": \"e8dbab85-98a4-49d4-94ad-b689a0da487b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T18:52:59.240619957Z\",\n      \"endTime\": \"2026-09-21T19:22:07.757736289Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"61c58512a4b1b02ae7341a4fbd482ea7169bc71e3540b1845f3f6f8c3517c8f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102095915643834368","occurred_at":"2026-09-21T18:01:20Z","failed_at":"alert","signature_id":"23aad75ee689d24c8d1008238b179160108eaca1183c0215f76a0cb565810f42","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-21T19:55:10Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j3834368-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-wfp5z\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-wfp5z (namespace arobit) on cluster ci01-j3834368-svc has averaged 126.5% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"bfca877d94fd30b669c37b79e9437c4f996244eb227f92f105218a5d97f7e38d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102095915643834368","occurred_at":"2026-09-21T18:01:20Z","failed_at":"alert","signature_id":"48fc36b4faaa65bd9e8e34d18123ea83691c098ed7a2b59b8c540ecea7c27cbd","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T19:14:35Z\n  Ended:       2026-09-21T19:47:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3834368-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.37:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3834368/providers/microsoft.monitor/accounts/hcps-j3834368\", namespace=\"ocm-arohcpci01-2sviu319g5e6he82ppjsp4nds4cimm8g-o7o2u6m5s2o5a7f\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e050989a-8cff-4b23-a05f-7aa2cc9bd30a\"\n  Description: Pod ocm-arohcpci01-2sviu319g5e6he82ppjsp4nds4cimm8g-o7o2u6m5s2o5a7f/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-21T19:14:35Z\n  Ended:       2026-09-21T19:48:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3834368-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.37:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3834368/providers/microsoft.monitor/accounts/hcps-j3834368\", namespace=\"ocm-arohcpci01-2sviu319g5e6he82ppjsp4nds4cimm8g-o7o2u6m5s2o5a7f\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e050989a-8cff-4b23-a05f-7aa2cc9bd30a\"\n  Description: Pod ocm-arohcpci01-2sviu319g5e6he82ppjsp4nds4cimm8g-o7o2u6m5s2o5a7f/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"ed8660e8af6f388bd44ee398af7cb976ed66c5dde3e07a8f6402e76e3b1bb945","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102095915643834368","occurred_at":"2026-09-21T18:01:20Z","failed_at":"alert","signature_id":"7bfb022c73ab6121802119bdc8fab30176d2abd6aed4c393c541ff8666d0374c","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T19:13:35Z\n  Ended:       2026-09-21T19:47:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3834368-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.37:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3834368/providers/microsoft.monitor/accounts/hcps-j3834368\", namespace=\"ocm-arohcpci01-2sviu319g5e6he82ppjsp4nds4cimm8g-o7o2u6m5s2o5a7f\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-hlfnw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sviu319g5e6he82ppjsp4nds4cimm8g-o7o2u6m5s2o5a7f/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-21T19:13:35Z\n  Ended:       2026-09-21T19:47:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3834368-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.37:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3834368/providers/microsoft.monitor/accounts/hcps-j3834368\", namespace=\"ocm-arohcpci01-2sviu319g5e6he82ppjsp4nds4cimm8g-o7o2u6m5s2o5a7f\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-hlfnw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sviu319g5e6he82ppjsp4nds4cimm8g-o7o2u6m5s2o5a7f/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102095907259420672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T18:01:12Z","started_at":"2026-09-21T18:01:12Z","completed_at":"2026-09-21T20:34:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T20:43:20.636969335Z","timing_metadata_checked_at":"2026-09-21T20:43:20.636969335Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T20:43:20.639717126Z","region_metadata_checked_at":"2026-09-21T20:43:20.639717126Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"6e6bb1401b4ae99a2cd976fc6b1c962f85f116b94bd598e64bc03287ef7b5787","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102095907259420672","occurred_at":"2026-09-21T18:01:12Z","failed_at":"e2e","signature_id":"d6d34872820247bb1e5071bfdbfac1e37fbb74c5be514065823665f538ab1c38","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001180120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-bjb7679qd9tc\" to finish deleting: GET https://rp.j9420672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/afef2232-ab40-40dc-a73c-ca40b7b2d291\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/afef2232-ab40-40dc-a73c-ca40b7b2d291\",\n      \"name\": \"afef2232-ab40-40dc-a73c-ca40b7b2d291\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T19:53:23.874456653Z\",\n      \"endTime\": \"2026-09-21T20:17:23.720551585Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svj30t1fitme77173ifoi1tkg3djlvm still exists (deletion dispatched at 2026-09-21T19:53:26Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001180120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-bjb7679qd9tc\" to finish deleting: GET https://rp.j9420672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/afef2232-ab40-40dc-a73c-ca40b7b2d291\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/afef2232-ab40-40dc-a73c-ca40b7b2d291\",\n      \"name\": \"afef2232-ab40-40dc-a73c-ca40b7b2d291\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T19:53:23.874456653Z\",\n      \"endTime\": \"2026-09-21T20:17:23.720551585Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svj30t1fitme77173ifoi1tkg3djlvm still exists (deletion dispatched at 2026-09-21T19:53:26Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b6a868c4e4b9da8f580820111ea1e1b5d022b229437ee4ba28ec6a565e5fd588","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102095907259420672","occurred_at":"2026-09-21T18:01:12Z","failed_at":"e2e","signature_id":"37f7d1dbf1718a5e1cd805e0562a8fabc7ecc1b737fb0e6e2e70225cf0fcbc8a","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00100e1f8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-gfsjs6\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-l9lptql2hbdl\" to finish deleting: GET https://rp.j9420672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8880f912-6c14-4f5f-a0fa-474232a52447\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8880f912-6c14-4f5f-a0fa-474232a52447\",\n      \"name\": \"8880f912-6c14-4f5f-a0fa-474232a52447\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T19:32:46.934009292Z\",\n      \"endTime\": \"2026-09-21T19:56:54.502530579Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svj34rj0a7r46h00e5iib5sgvik5kud still exists (deletion dispatched at 2026-09-21T19:32:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00100e1f8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-gfsjs6\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-l9lptql2hbdl\" to finish deleting: GET https://rp.j9420672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8880f912-6c14-4f5f-a0fa-474232a52447\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8880f912-6c14-4f5f-a0fa-474232a52447\",\n      \"name\": \"8880f912-6c14-4f5f-a0fa-474232a52447\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T19:32:46.934009292Z\",\n      \"endTime\": \"2026-09-21T19:56:54.502530579Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svj34rj0a7r46h00e5iib5sgvik5kud still exists (deletion dispatched at 2026-09-21T19:32:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e88309dc07e57059d345b5f3793126d316e7546e17f77b16ce8fe3e89a30dc72","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102095907259420672","occurred_at":"2026-09-21T18:01:12Z","failed_at":"alert","signature_id":"9f0083bb350ed4104f1dd24f927a2773084ee0e1ba3d46a163c961193ef03afe","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-21T20:12:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j9420672-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-t8rc5\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-t8rc5 (namespace arobit) on cluster ci01-j9420672-svc has averaged 126.3% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7050/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102089024922456064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7050,"pr_state":"closed","pr_sha":"cf72712284cb479e9bca6f86952f68b72255a389","final_merged_sha":"cf72712284cb479e9bca6f86952f68b72255a389","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T17:27:02Z","started_at":"2026-09-21T17:27:02Z","completed_at":"2026-09-21T18:12:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T18:15:56.057237839Z","timing_metadata_checked_at":"2026-09-21T18:15:56.057237839Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T18:15:56.063452419Z","region_metadata_checked_at":"2026-09-21T18:15:56.063452419Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"f3eba606ca6b552306e81a9fc724d65951f3d72d976077bc72465c8ea7804b0b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7050/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102089024922456064","occurred_at":"2026-09-21T17:27:02Z","failed_at":"provision","signature_id":"2a1421a5921203c83346dd78ed94d1d8c9de6e668bb1a446c3d23688c2d64bec","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/admin-api-cert","test_suite":"templatize-pipeline","failure_text":"time=2026-09-21T18:05:06.874Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert description=\"Step admin-api-cert\\n Kind: CreateCertificate\\n\"\ntime=2026-09-21T18:05:06.874Z level=INFO msg=\"Creating certificate\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert vaultBaseUrl=https://aro-hcp-dev-svc-kv.vault.azure.net/ certificateName=admin-api-cert-ci01-j2456064 issuer=Self commonName=admin.j2456064.hcpsvc.osadev.cloud san=admin.j2456064.hcpsvc.osadev.cloud\ntime=2026-09-21T18:05:07.523Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert err=\"error running Create Certificate Step: failed to create certificate \\\"admin-api-cert-ci01-j2456064\\\" in vault \\\"https://aro-hcp-dev-svc-kv.vault.azure.net/\\\": POST https://aro-hcp-dev-svc-kv.vault.azure.net/certificates/admin-api-cert-ci01-j2456064/create\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Certificate admin-api-cert-ci01-j2456064 is currently in a deleted but recoverable state, and its name cannot be reused; in this state, the certificate can only be recovered or purged.\\\",\\n    \\\"innererror\\\": {\\n      \\\"code\\\": \\\"ObjectIsDeletedButRecoverable\\\"\\n    }\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6f6e7e66ad8bc42bd6647f1e246b1e77205441e0c9495974d9b1f940d8bdec9c","failure_pattern":"ERROR CODE: Conflict; detail code ObjectIsDeletedButRecoverable","search_query":"ERROR CODE: Conflict"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6f6e7e66ad8bc42bd6647f1e246b1e77205441e0c9495974d9b1f940d8bdec9c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7048/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102086108664827904","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7048,"pr_state":"closed","pr_sha":"3ba2a23f7154317c2d962d0d1a8d55ecbaa88ce3","final_merged_sha":"3ba2a23f7154317c2d962d0d1a8d55ecbaa88ce3","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T17:04:46Z","started_at":"2026-09-21T17:04:46Z","completed_at":"2026-09-21T19:48:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T19:50:46.695922936Z","timing_metadata_checked_at":"2026-09-21T19:50:46.695922936Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T19:50:46.70066612Z","region_metadata_checked_at":"2026-09-21T19:50:46.70066612Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"22cec999456efb96095ff064e3f0589eccde87b9b11defb8ffdb915ab8169998","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7048/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102086108664827904","occurred_at":"2026-09-21T17:04:46Z","failed_at":"alert","signature_id":"5895af5bb01f04c8a2312737be8744a64489c84b607b638ccdfa4b71329c85f3","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-21T19:28:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j4827904-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-896nm\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-896nm (namespace arobit) on cluster ci01-j4827904-svc has averaged 126.5% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"3cde67230b3d5637cfc3be1392cce5719829398004b74712d4a119613adefb18","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7048/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102086108664827904","occurred_at":"2026-09-21T17:04:46Z","failed_at":"e2e","signature_id":"6dfb1b2941b0e61b473458ba9e292338996b9a5a671b9dcbed59a3689da29f41","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011862b8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-22-smvkdn\" in resourcegroup=\"rg-zstream-upgrade-4-22-4vx75r2hvn8k\" to finish deleting: GET https://rp.j4827904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e876a434-7627-44d5-9031-9e251707af04\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e876a434-7627-44d5-9031-9e251707af04\",\n      \"name\": \"e876a434-7627-44d5-9031-9e251707af04\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T18:53:46.407451527Z\",\n      \"endTime\": \"2026-09-21T19:17:49.378648262Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svifgvg503o4o6si36j2kqisb1ahf6r still exists (deletion dispatched at 2026-09-21T18:53:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011862b8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-22-smvkdn\" in resourcegroup=\"rg-zstream-upgrade-4-22-4vx75r2hvn8k\" to finish deleting: GET https://rp.j4827904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e876a434-7627-44d5-9031-9e251707af04\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e876a434-7627-44d5-9031-9e251707af04\",\n      \"name\": \"e876a434-7627-44d5-9031-9e251707af04\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T18:53:46.407451527Z\",\n      \"endTime\": \"2026-09-21T19:17:49.378648262Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svifgvg503o4o6si36j2kqisb1ahf6r still exists (deletion dispatched at 2026-09-21T18:53:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a6c3b0680bc37260b8a2e80449b711ce70025dd045dd83fc0aa46b23d65e3cd7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7048/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102086108664827904","occurred_at":"2026-09-21T17:04:46Z","failed_at":"e2e","signature_id":"ba3a929d8e0f5a3f9ac26a138b1b0ffac23aaa7a310ce650e182880ce3852d06","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0015a01e0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-n5qpxr9tgd2d\" to finish deleting: GET https://rp.j4827904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/561c8dc5-4b66-4304-8d8d-3b664fdd9fa3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/561c8dc5-4b66-4304-8d8d-3b664fdd9fa3\",\n      \"name\": \"561c8dc5-4b66-4304-8d8d-3b664fdd9fa3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T19:07:44.438691737Z\",\n      \"endTime\": \"2026-09-21T19:31:49.427520234Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svige89olceu0kb3nu2q8p0dprdpv6e still exists (deletion dispatched at 2026-09-21T19:07:46Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0015a01e0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-n5qpxr9tgd2d\" to finish deleting: GET https://rp.j4827904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/561c8dc5-4b66-4304-8d8d-3b664fdd9fa3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/561c8dc5-4b66-4304-8d8d-3b664fdd9fa3\",\n      \"name\": \"561c8dc5-4b66-4304-8d8d-3b664fdd9fa3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T19:07:44.438691737Z\",\n      \"endTime\": \"2026-09-21T19:31:49.427520234Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svige89olceu0kb3nu2q8p0dprdpv6e still exists (deletion dispatched at 2026-09-21T19:07:46Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e5635cb5387695ea16a3860871ed2c10a72770026c54c1a60d92b69e1ea9948a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7048/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102086108664827904","occurred_at":"2026-09-21T17:04:46Z","failed_at":"e2e","signature_id":"d5edfd1facf82b14cb15c4a50e36f171b9f1d024d245acc97bb843fe2755e899","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f90108\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-48vths\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-275glcwxw5f5\" to finish deleting: GET https://rp.j4827904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d275da87-5f27-4c2d-98d4-e3383c6edfc5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d275da87-5f27-4c2d-98d4-e3383c6edfc5\",\n      \"name\": \"d275da87-5f27-4c2d-98d4-e3383c6edfc5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T18:50:16.734965807Z\",\n      \"endTime\": \"2026-09-21T19:14:22.065872144Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sviffoqouvutp8jj3visd57vuq81o6o still exists (deletion dispatched at 2026-09-21T18:50:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f90108\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-48vths\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-275glcwxw5f5\" to finish deleting: GET https://rp.j4827904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d275da87-5f27-4c2d-98d4-e3383c6edfc5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d275da87-5f27-4c2d-98d4-e3383c6edfc5\",\n      \"name\": \"d275da87-5f27-4c2d-98d4-e3383c6edfc5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T18:50:16.734965807Z\",\n      \"endTime\": \"2026-09-21T19:14:22.065872144Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sviffoqouvutp8jj3visd57vuq81o6o still exists (deletion dispatched at 2026-09-21T18:50:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7042/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102083142897635328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7042,"pr_state":"closed","pr_sha":"6a2ceb3234624c8807be9fd61b93ac14e967a5cf","final_merged_sha":"ed561d7657bd4aae970254bd806604f0b5a59888","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T16:54:04Z","started_at":"2026-09-21T16:54:04Z","completed_at":"2026-09-21T18:19:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T18:26:43.132222768Z","timing_metadata_checked_at":"2026-09-21T18:26:43.132222768Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T18:26:43.135445658Z","region_metadata_checked_at":"2026-09-21T18:26:43.135445658Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"a449dd6af21a1358ddee533a8c5be2d4a3c7d8107ae5f8986d84b8ee29621536","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7042/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102083142897635328","occurred_at":"2026-09-21T16:54:04Z","failed_at":"provision","signature_id":"ca42abb0330325a4ac71c384a5a0eca4847918bb6bd88e42a0e216b1c3a6c5c3","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-21T18:04:56.091Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-09-21T18:12:34.654Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-21T18:04:56.09869494Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7635328-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7635328-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-09-21T18:04:58.120073331Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7635328-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7635328-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-21T18:04:58.527982819Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7635328-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7635328-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-21T18:12:34.652011621Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7635328-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7635328-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"89CF033DB1DF73D0\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7635328-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7635328-mgmt-2/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j7635328-mgmt-2/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-21T18:10:26.0503797Z\\\",\\\"duration\\\":\\\"PT1M56.3405526S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-21T18:12:34.652086242Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7635328-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7635328-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"A0DB23EC9205187A\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7635328-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7635328-mgmt-2/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j7635328-mgmt-2/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-21T18:10:26.0503797Z\\\",\\\"duration\\\":\\\"PT1M39.5985567S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-21T18:12:34.652107413Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7635328-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7635328-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"977918EE1130A569\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7635328-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7635328-mgmt-2/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j7635328-mgmt-2/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-21T18:10:26.0503797Z\\\",\\\"duration\\\":\\\"PT4.3101609S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 32 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-21T18:12:34.652152984Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7635328-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7635328-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"5835781ECA9C15E0\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7635328-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7635328-mgmt-2/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j7635328-mgmt-2/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-21T18:05:03.7378518Z\\\",\\\"duration\\\":\\\"PT5M26.5601897S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 32 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-21T18:12:34.652204585Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7635328-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7635328-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"E787274800217612\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7635328-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7635328-mgmt-2/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j7635328-mgmt-2/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-21T18:10:26.0503797Z\\\",\\\"duration\\\":\\\"PT4.0601636S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 32 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-21T18:12:34.652246526Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7635328-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7635328-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"B468A793AB498001\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7635328-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7635328-mgmt-2\\\",\\\"resourceName\\\":\\\"ci01-j7635328-mgmt-2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-21T18:05:03.7378518Z\\\",\\\"duration\\\":\\\"PT5M22.2477188S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-21T18:12:34.652279217Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7635328-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7635328-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7635328-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584115929865651702\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7635328-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n    \\\\\\\"details\\\\\\\": [\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7635328-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7635328-mgmt-2/agentPools/userswft3\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 32 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n      },\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7635328-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7635328-mgmt-2/agentPools/userswft1\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 32 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n      },\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7635328-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7635328-mgmt-2/agentPools/userswft2\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 32 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n      }\\\\n    ]\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"7m38.553591047s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b821a370667d0a6279446842808eaf289925d243f648908c207a98a9eafaa561","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ErrCode_InsufficientVCPUQuota; detail message Insufficient vcpu quota requested \u003ccount\u003e, remaining \u003ccount\u003e for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: \u003curl\u003e Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b821a370667d0a6279446842808eaf289925d243f648908c207a98a9eafaa561"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102079067309215744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6359,"pr_state":"open","pr_sha":"f72c915cb5373981522678b6346b4f7fee96c35f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T16:50:18Z","started_at":"2026-09-21T16:50:18Z","completed_at":"2026-09-21T19:05:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T19:09:08.234273031Z","timing_metadata_checked_at":"2026-09-21T19:09:08.234273031Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T19:09:08.236867823Z","region_metadata_checked_at":"2026-09-21T19:09:08.236867823Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"0d9dfa589e67c1092ed20c86741b6d54021e3293ef30377790e7e91629d0cdc2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102079067309215744","occurred_at":"2026-09-21T16:50:18Z","failed_at":"e2e","signature_id":"9ce27d9539e159cea945e154049733cc95a45868228d102cebe77c5beb734973","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001936408\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-noedge-psshqd\" in resourcegroup=\"rg-np-noedge-psshqd-ncsrs64d4wtf\" to finish deleting: GET https://rp.j9215744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1aa78a4-eda6-4dfe-a9f6-210a2c907bf2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1aa78a4-eda6-4dfe-a9f6-210a2c907bf2\",\n      \"name\": \"a1aa78a4-eda6-4dfe-a9f6-210a2c907bf2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T18:11:33.824755345Z\",\n      \"endTime\": \"2026-09-21T18:35:38.208437669Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svhqdm2nth1em7m38u2a3ckpeov7mb3 still exists (deletion dispatched at 2026-09-21T18:11:35Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001936408\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-noedge-psshqd\" in resourcegroup=\"rg-np-noedge-psshqd-ncsrs64d4wtf\" to finish deleting: GET https://rp.j9215744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1aa78a4-eda6-4dfe-a9f6-210a2c907bf2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1aa78a4-eda6-4dfe-a9f6-210a2c907bf2\",\n      \"name\": \"a1aa78a4-eda6-4dfe-a9f6-210a2c907bf2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T18:11:33.824755345Z\",\n      \"endTime\": \"2026-09-21T18:35:38.208437669Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svhqdm2nth1em7m38u2a3ckpeov7mb3 still exists (deletion dispatched at 2026-09-21T18:11:35Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"62e5fe3ef93ef7552da6d40f4e3273c68cf77e8f85795cc20180f0332682b516","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102079067309215744","occurred_at":"2026-09-21T16:50:18Z","failed_at":"alert","signature_id":"53d18c679af365c81726ad04e6f2e68d663c413ffef032b73a7a59fb1aa0b54c","test_name":"[aro-hcp-observability] [hcp] alert userJourneySwiftRouterAvailability1h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T17:49:18Z\n  Ended:       2026-09-21T17:59:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneySwiftRouterAvailability1h5m\", burn_rate_tier=\"fast\", cluster=\"ci01-j9215744-mgmt-2\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9215744/providers/microsoft.monitor/accounts/hcps-j9215744\", region=\"westus3\", severity=\"3\", short_window=\"5m\"\n  Description: SWIFT router-capacity availability is consuming its error budget at the fast burn rate.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"27efdba0688c010a9407237cceb9d29c447c7f9bd7965cf5f46a9ce24313bf72","failure_pattern":"alert [hcp] userJourneySwiftRouterAvailability1h5m fired","search_query":"alert [hcp] userJourneySwiftRouterAvailability1h5m fired"}},{"row_id":"67236863be5e491ac00f8c12f7629d75e8dd02e7481406cdb002f1a396713f22","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102079067309215744","occurred_at":"2026-09-21T16:50:18Z","failed_at":"e2e","signature_id":"e44960c13ea73395f0b50672831157838cf129e9acfc13b40c31d397af6238b1","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc002f1e258\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-tcs5bmz8lgph\" to finish deleting: GET https://rp.j9215744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b20683e8-2db8-4471-8a6c-28cbad3b843e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b20683e8-2db8-4471-8a6c-28cbad3b843e\",\n      \"name\": \"b20683e8-2db8-4471-8a6c-28cbad3b843e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T18:10:35.751034396Z\",\n      \"endTime\": \"2026-09-21T18:34:39.971957701Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svhqd1btvndrf1on9ohms60mrn6ure9 still exists (deletion dispatched at 2026-09-21T18:10:36Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc002f1e258\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-tcs5bmz8lgph\" to finish deleting: GET https://rp.j9215744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b20683e8-2db8-4471-8a6c-28cbad3b843e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b20683e8-2db8-4471-8a6c-28cbad3b843e\",\n      \"name\": \"b20683e8-2db8-4471-8a6c-28cbad3b843e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T18:10:35.751034396Z\",\n      \"endTime\": \"2026-09-21T18:34:39.971957701Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svhqd1btvndrf1on9ohms60mrn6ure9 still exists (deletion dispatched at 2026-09-21T18:10:36Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6ef5e169bcd528762f220a5936556db61a9925ac1511e3d91dfaa336be35ca6d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102079067309215744","occurred_at":"2026-09-21T16:50:18Z","failed_at":"e2e","signature_id":"dd5471b4fe6162ffe643013fbf461da22d1bd65d515c46574c7d9558eb35b045","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001772240\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-rwzb5rcs4pj8\" to finish deleting: GET https://rp.j9215744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75a07b5a-1ded-47da-8678-542908c9bc97\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75a07b5a-1ded-47da-8678-542908c9bc97\",\n      \"name\": \"75a07b5a-1ded-47da-8678-542908c9bc97\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T18:01:00.48082139Z\",\n      \"endTime\": \"2026-09-21T18:25:09.961424795Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svhqhma5bvus8lp33ueu8h6ne8kem9i still exists (deletion dispatched at 2026-09-21T18:01:05Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001772240\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-rwzb5rcs4pj8\" to finish deleting: GET https://rp.j9215744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75a07b5a-1ded-47da-8678-542908c9bc97\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75a07b5a-1ded-47da-8678-542908c9bc97\",\n      \"name\": \"75a07b5a-1ded-47da-8678-542908c9bc97\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T18:01:00.48082139Z\",\n      \"endTime\": \"2026-09-21T18:25:09.961424795Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svhqhma5bvus8lp33ueu8h6ne8kem9i still exists (deletion dispatched at 2026-09-21T18:01:05Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a5dc9e5ae8520dc209b812899b01d431a7f94eccdfac367270d2dd6e3cf8a90b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102079067309215744","occurred_at":"2026-09-21T16:50:18Z","failed_at":"e2e","signature_id":"c5768b51e4f5ce2da45fbe6c9025fe9018a3d5ef626d16a5dbeb09f49456339a","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001ba4090\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-srxbhlsqx7qn\" to finish deleting: GET https://rp.j9215744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1d29fb72-28fe-4cfe-a5a5-e0e9ab6ec12d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1d29fb72-28fe-4cfe-a5a5-e0e9ab6ec12d\",\n      \"name\": \"1d29fb72-28fe-4cfe-a5a5-e0e9ab6ec12d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T18:23:10.51256016Z\",\n      \"endTime\": \"2026-09-21T18:47:18.263127245Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svhqg1ou1ikf2se3mpk6pla50p8jnev still exists (deletion dispatched at 2026-09-21T18:23:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001ba4090\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-srxbhlsqx7qn\" to finish deleting: GET https://rp.j9215744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1d29fb72-28fe-4cfe-a5a5-e0e9ab6ec12d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1d29fb72-28fe-4cfe-a5a5-e0e9ab6ec12d\",\n      \"name\": \"1d29fb72-28fe-4cfe-a5a5-e0e9ab6ec12d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T18:23:10.51256016Z\",\n      \"endTime\": \"2026-09-21T18:47:18.263127245Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svhqg1ou1ikf2se3mpk6pla50p8jnev still exists (deletion dispatched at 2026-09-21T18:23:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fbd24f25cdd97d5bce8c6159d79ff0243aac64c8f3210ff91694337b5d7ebc30","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102079067309215744","occurred_at":"2026-09-21T16:50:18Z","failed_at":"alert","signature_id":"4c9ebf55646d9822d2ba136374e18c65f33ed4e8162d6b5a55485d1ce330e144","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-21T18:52:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j9215744-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-2kc9s\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-2kc9s (namespace arobit) on cluster ci01-j9215744-svc has averaged 127% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["27efdba0688c010a9407237cceb9d29c447c7f9bd7965cf5f46a9ce24313bf72","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7049/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102076044226859008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7049,"pr_state":"open","pr_sha":"e4c4bdc131c40f378c18666ee36151f8170dc0ad","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T16:38:49Z","started_at":"2026-09-21T16:38:49Z","completed_at":"2026-09-21T18:51:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T18:58:52.914643043Z","timing_metadata_checked_at":"2026-09-21T18:58:52.914643043Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T18:58:52.917251636Z","region_metadata_checked_at":"2026-09-21T18:58:52.917251636Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"38731ba8e44f8f81aa5fd7db18db7fe2460a46e250fc92908d71ee5bc8594c8a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7049/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102076044226859008","occurred_at":"2026-09-21T16:38:49Z","failed_at":"e2e","signature_id":"8dcbb138246cfee8e75fb9b936e0ef04b2b3011c75a61e9a8f0766f9c310762f","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000012348\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-22-4cb4zq\" in resourcegroup=\"rg-zstream-upgrade-4-22-pc258q6wx68w\" to finish deleting: GET https://rp.j6859008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b75a99c-ac2f-440b-ac6f-c4c243c95b28\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b75a99c-ac2f-440b-ac6f-c4c243c95b28\",\n      \"name\": \"5b75a99c-ac2f-440b-ac6f-c4c243c95b28\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T17:51:47.024707511Z\",\n      \"endTime\": \"2026-09-21T18:15:47.504998786Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svhklpduudqefot27ncancnb90le46e still exists (deletion dispatched at 2026-09-21T17:51:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000012348\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-22-4cb4zq\" in resourcegroup=\"rg-zstream-upgrade-4-22-pc258q6wx68w\" to finish deleting: GET https://rp.j6859008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b75a99c-ac2f-440b-ac6f-c4c243c95b28\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b75a99c-ac2f-440b-ac6f-c4c243c95b28\",\n      \"name\": \"5b75a99c-ac2f-440b-ac6f-c4c243c95b28\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T17:51:47.024707511Z\",\n      \"endTime\": \"2026-09-21T18:15:47.504998786Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svhklpduudqefot27ncancnb90le46e still exists (deletion dispatched at 2026-09-21T17:51:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b548ac782dc70685e1548ca6e6fdaba796684adc2cfdfcc088834ace86a4b76b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7049/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102076044226859008","occurred_at":"2026-09-21T16:38:49Z","failed_at":"alert","signature_id":"b08961e9d2b8dd97f5005b3e4f567ae665c6086f801434ad71caef7cf0bc0d52","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T17:57:16Z\n  Ended:       2026-09-21T18:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6859008-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6859008-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"fbddff0ec746bdd335fde9f6adce9d1439171cad0b0f477850704e0b640481a1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7049/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102076044226859008","occurred_at":"2026-09-21T16:38:49Z","failed_at":"e2e","signature_id":"f1caa39914325fbf899ad353579652078b7871b78754ddd3ec1ab4ac66cfaac7","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00188e0d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-9phd7h2t65cd\" to finish deleting: GET https://rp.j6859008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b69f6ec8-f07c-4d1f-aeb6-4eb7e1a660e8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b69f6ec8-f07c-4d1f-aeb6-4eb7e1a660e8\",\n      \"name\": \"b69f6ec8-f07c-4d1f-aeb6-4eb7e1a660e8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T18:09:34.803739999Z\",\n      \"endTime\": \"2026-09-21T18:33:43.452123986Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svhkp33ena07l04uacaqd77ag09f6lb still exists (deletion dispatched at 2026-09-21T18:09:35Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00188e0d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-9phd7h2t65cd\" to finish deleting: GET https://rp.j6859008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b69f6ec8-f07c-4d1f-aeb6-4eb7e1a660e8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b69f6ec8-f07c-4d1f-aeb6-4eb7e1a660e8\",\n      \"name\": \"b69f6ec8-f07c-4d1f-aeb6-4eb7e1a660e8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T18:09:34.803739999Z\",\n      \"endTime\": \"2026-09-21T18:33:43.452123986Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svhkp33ena07l04uacaqd77ag09f6lb still exists (deletion dispatched at 2026-09-21T18:09:35Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102075143474909184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6925,"pr_state":"open","pr_sha":"9ef5982dac7a7b22811f15404810c31d0372579b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T16:38:06Z","started_at":"2026-09-21T16:38:06Z","completed_at":"2026-09-21T18:46:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T18:58:50.306761765Z","timing_metadata_checked_at":"2026-09-21T18:58:50.306761765Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T18:58:50.309165958Z","region_metadata_checked_at":"2026-09-21T18:58:50.309165958Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"d2f82564dcb34d7e1392fbfe7acc6adb430e5bb5626fc57bdd8246c15992c85d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102075143474909184","occurred_at":"2026-09-21T16:38:06Z","failed_at":"alert","signature_id":"ea9ae21cd686d5c62d1c47948511fd85e6089023a32829db2e924103b90a40b2","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T17:59:28Z\n  Ended:       2026-09-21T18:22:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4909184-svc\", component=\"frontend\", method=\"get\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} has exceeded 250ms over the past 30 minutes (cluster ci01-j4909184-svc).\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-21T18:00:27Z\n  Ended:       2026-09-21T18:07:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4909184-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4909184-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102072370310156288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5846,"pr_state":"closed","pr_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","final_merged_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T16:24:04Z","started_at":"2026-09-21T16:24:04Z","completed_at":"2026-09-21T18:49:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T18:58:50.211365947Z","timing_metadata_checked_at":"2026-09-21T18:58:50.211365947Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T18:58:50.214207739Z","region_metadata_checked_at":"2026-09-21T18:58:50.214207739Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"52ffe2cf0494d1d55b101c20345e820382afaab990148548c2a85f1926871e12","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102072370310156288","occurred_at":"2026-09-21T16:24:04Z","failed_at":"alert","signature_id":"348f0f120c2c736939bdc525786464c8c9dcdf284d2fd999de40fcc6a93e1625","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T17:31:08Z\n  Ended:       2026-09-21T18:02:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j0156288-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j0156288-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"7262eea29fc1c8b9d4d5b2279d8802bb8bf057fb72b7ddc9d90e6eacd26903cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102072370310156288","occurred_at":"2026-09-21T16:24:04Z","failed_at":"e2e","signature_id":"3fa123edaa22783b2efbc9f089dfb0c0af0f9f57c61dda43f1c9f7848b8c06aa","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001158f30\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-wj9jcwbrzxr4\" to finish deleting: GET https://rp.j0156288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa4034c8-2f0e-4b61-b9da-9550bb20be12\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa4034c8-2f0e-4b61-b9da-9550bb20be12\",\n      \"name\": \"aa4034c8-2f0e-4b61-b9da-9550bb20be12\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T17:53:44.019609975Z\",\n      \"endTime\": \"2026-09-21T18:17:44.806286193Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svhjni2k5j3efk09d35q7o3jl60ii54 still exists (deletion dispatched at 2026-09-21T17:53:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001158f30\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-wj9jcwbrzxr4\" to finish deleting: GET https://rp.j0156288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa4034c8-2f0e-4b61-b9da-9550bb20be12\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa4034c8-2f0e-4b61-b9da-9550bb20be12\",\n      \"name\": \"aa4034c8-2f0e-4b61-b9da-9550bb20be12\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T17:53:44.019609975Z\",\n      \"endTime\": \"2026-09-21T18:17:44.806286193Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svhjni2k5j3efk09d35q7o3jl60ii54 still exists (deletion dispatched at 2026-09-21T17:53:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9d717e6deb3cdd2bb485e820f20b240cab0a086467bbed9d5f4fbcc61964adfa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102072370310156288","occurred_at":"2026-09-21T16:24:04Z","failed_at":"e2e","signature_id":"f98fb9a40f84f14d6c28a914e9795fd74cc4b5c8b0e23b8bbfdab978bade3cde","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc002c141b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-noedge-b2f487\" in resourcegroup=\"rg-np-noedge-b2f487-6bxvkg6776wt\" to finish deleting: GET https://rp.j0156288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/06a6669b-9b5f-4dc0-bd48-88c4751af6c1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/06a6669b-9b5f-4dc0-bd48-88c4751af6c1\",\n      \"name\": \"06a6669b-9b5f-4dc0-bd48-88c4751af6c1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T18:02:40.422855986Z\",\n      \"endTime\": \"2026-09-21T18:26:48.80366423Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svhjefbqorrcdjnfgprhg85cj8u08hr still exists (deletion dispatched at 2026-09-21T18:02:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc002c141b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-noedge-b2f487\" in resourcegroup=\"rg-np-noedge-b2f487-6bxvkg6776wt\" to finish deleting: GET https://rp.j0156288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/06a6669b-9b5f-4dc0-bd48-88c4751af6c1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/06a6669b-9b5f-4dc0-bd48-88c4751af6c1\",\n      \"name\": \"06a6669b-9b5f-4dc0-bd48-88c4751af6c1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T18:02:40.422855986Z\",\n      \"endTime\": \"2026-09-21T18:26:48.80366423Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svhjefbqorrcdjnfgprhg85cj8u08hr still exists (deletion dispatched at 2026-09-21T18:02:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ac275cc2cfb9bce9230a09101f4b84f09ea75d602547e419938ef08e64e196bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102072370310156288","occurred_at":"2026-09-21T16:24:04Z","failed_at":"e2e","signature_id":"df3a5f750a26df15ce62c2b547731e719daaf59b9ca008fa168fa6a9a1988811","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001d942a0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2fdhlxl78lrz\" to finish deleting: GET https://rp.j0156288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3430e65f-cc40-47ef-a0db-ce713c3cc263\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3430e65f-cc40-47ef-a0db-ce713c3cc263\",\n      \"name\": \"3430e65f-cc40-47ef-a0db-ce713c3cc263\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T17:53:36.253164862Z\",\n      \"endTime\": \"2026-09-21T18:17:44.78393138Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svhjfl6abarq37olncjgsqioudrthms still exists (deletion dispatched at 2026-09-21T17:53:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001d942a0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2fdhlxl78lrz\" to finish deleting: GET https://rp.j0156288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3430e65f-cc40-47ef-a0db-ce713c3cc263\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3430e65f-cc40-47ef-a0db-ce713c3cc263\",\n      \"name\": \"3430e65f-cc40-47ef-a0db-ce713c3cc263\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T17:53:36.253164862Z\",\n      \"endTime\": \"2026-09-21T18:17:44.78393138Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svhjfl6abarq37olncjgsqioudrthms still exists (deletion dispatched at 2026-09-21T17:53:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b66c9764035f0d9c1e6f75855e45ee076892d06df61ea88d985eaf22b6cfba1e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102072370310156288","occurred_at":"2026-09-21T16:24:04Z","failed_at":"alert","signature_id":"22c5582a22d1e57db7d9e5433ed7d075c7070bd9f8c20d549fdb304794b969f5","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-21T18:30:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j0156288-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-xzz74\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-xzz74 (namespace arobit) on cluster ci01-j0156288-svc has averaged 125.9% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7018/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102086834673684480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7018,"pr_state":"open","pr_sha":"5c4e7b27f7d76d1aac25b9e8c9d004c3a38882f4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-21T16:02:09Z","started_at":"2026-09-21T16:02:09Z","completed_at":"2026-09-21T19:52:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T20:01:34.035900502Z","timing_metadata_checked_at":"2026-09-21T20:01:34.035900502Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T20:01:34.038974492Z","region_metadata_checked_at":"2026-09-21T20:01:34.038974492Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102076823893446656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6945,"pr_state":"closed","pr_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","final_merged_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T15:50:23Z","started_at":"2026-09-21T15:50:23Z","completed_at":"2026-09-21T18:50:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T18:58:50.43662038Z","timing_metadata_checked_at":"2026-09-21T18:58:50.43662038Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T18:58:50.439342972Z","region_metadata_checked_at":"2026-09-21T18:58:50.439342972Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"1e0d0bc7ced9f7f4636bafff18addd9cf8b55fed7c474e0cdcb3119b3384ea46","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102076823893446656","occurred_at":"2026-09-21T15:50:23Z","failed_at":"e2e","signature_id":"abc92724102639ab27110ae804c6b65de07bff0c744bc44b5caefe3b777bb75d","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001d82240\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-2mzm67n9dg7h\" to finish deleting: GET https://rp.j3446656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ac1872f-d57b-49ad-bcd6-f3c83e6ea369\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ac1872f-d57b-49ad-bcd6-f3c83e6ea369\",\n      \"name\": \"8ac1872f-d57b-49ad-bcd6-f3c83e6ea369\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T18:01:54.77134414Z\",\n      \"endTime\": \"2026-09-21T18:26:00.13640441Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svhm5ulmhqhmin15g8i2o747lr57t82 still exists (deletion dispatched at 2026-09-21T18:01:55Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001d82240\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-2mzm67n9dg7h\" to finish deleting: GET https://rp.j3446656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ac1872f-d57b-49ad-bcd6-f3c83e6ea369\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ac1872f-d57b-49ad-bcd6-f3c83e6ea369\",\n      \"name\": \"8ac1872f-d57b-49ad-bcd6-f3c83e6ea369\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T18:01:54.77134414Z\",\n      \"endTime\": \"2026-09-21T18:26:00.13640441Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svhm5ulmhqhmin15g8i2o747lr57t82 still exists (deletion dispatched at 2026-09-21T18:01:55Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"533fadd172ce8be388731fbb1a240ae5927bc508cd5c2ceb5765254a7f1ac3ad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102076823893446656","occurred_at":"2026-09-21T15:50:23Z","failed_at":"alert","signature_id":"3ecd69de3857dae3b2127f6e6f526fa8817a80c8e8e942bb9e9d41a170f9cb7e","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T17:45:44Z\n  Ended:       2026-09-21T18:08:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3446656-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3446656-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102065610849521664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6688,"pr_state":"open","pr_sha":"8a79a12a7517643489de8765e75a2ba51e3b6996","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T15:45:40Z","started_at":"2026-09-21T15:45:40Z","completed_at":"2026-09-21T18:18:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T18:26:43.009034764Z","timing_metadata_checked_at":"2026-09-21T18:26:43.009034764Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T18:26:43.014683346Z","region_metadata_checked_at":"2026-09-21T18:26:43.014683346Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"47432004c9cca9861f924bc8aa8c26dc14b5da3f8d95f05f7a89fe0960a1468a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102065610849521664","occurred_at":"2026-09-21T15:45:40Z","failed_at":"e2e","signature_id":"b9486bf9893d4610752968225ca23223bc0af5ffd8971a78dd17f145b4ae1633","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc00125b6a0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-5248mpsj56q6\" to finish creating: GET https://rp.j9521664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0cd0358b-031c-46ab-9abd-cd357255bda0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0cd0358b-031c-46ab-9abd-cd357255bda0\",\n      \"name\": \"0cd0358b-031c-46ab-9abd-cd357255bda0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T17:03:39.705515335Z\",\n      \"endTime\": \"2026-09-21T17:22:46.542422976Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc00125b6a0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-5248mpsj56q6\" to finish creating: GET https://rp.j9521664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0cd0358b-031c-46ab-9abd-cd357255bda0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0cd0358b-031c-46ab-9abd-cd357255bda0\",\n      \"name\": \"0cd0358b-031c-46ab-9abd-cd357255bda0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T17:03:39.705515335Z\",\n      \"endTime\": \"2026-09-21T17:22:46.542422976Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-5248mpsj56q6\" to finish creating: GET https://rp"}},{"row_id":"5565f21790c970175b09046fd04807cbe7ec66f0273dc3a74f1f220713e468a0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102065610849521664","occurred_at":"2026-09-21T15:45:40Z","failed_at":"e2e","signature_id":"9934578074a51747b41efab649903f34e73db709396b8526d1a5bd5b86154e6b","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00100e260\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-mqf2gjsr2md2\" to finish creating: GET https://rp.j9521664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b8a1157-dcd4-4bd7-b89a-f865c271214f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b8a1157-dcd4-4bd7-b89a-f865c271214f\",\n      \"name\": \"5b8a1157-dcd4-4bd7-b89a-f865c271214f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T17:01:12.189816506Z\",\n      \"endTime\": \"2026-09-21T17:20:16.669273518Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00100e260\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-mqf2gjsr2md2\" to finish creating: GET https://rp.j9521664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b8a1157-dcd4-4bd7-b89a-f865c271214f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b8a1157-dcd4-4bd7-b89a-f865c271214f\",\n      \"name\": \"5b8a1157-dcd4-4bd7-b89a-f865c271214f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T17:01:12.189816506Z\",\n      \"endTime\": \"2026-09-21T17:20:16.669273518Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a6cc272cff8b4e84c8e8fe098192859245d300f89f991171de2deb392779175b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102065610849521664","occurred_at":"2026-09-21T15:45:40Z","failed_at":"alert","signature_id":"e621adba2e76961d7fe1fedea9d941d015f07a1e6006c8f6ba9ab165d9d59ea8","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-21T18:02:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j9521664-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-bqdp4\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-bqdp4 (namespace arobit) on cluster ci01-j9521664-svc has averaged 125.5% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"f50723e7489e6725100b67ab174f234f4b6fd1c16388cb5b9e1a99a9b30a4abf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102065610849521664","occurred_at":"2026-09-21T15:45:40Z","failed_at":"e2e","signature_id":"136fff7ced212207b5927920916e769bdf0e0e55f68574094087b63709bda933","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a0e198\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-9vcphdzd6v79\" to finish deleting: GET https://rp.j9521664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e06ebcd7-3269-46a4-b719-7aa1cfcbf3f8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e06ebcd7-3269-46a4-b719-7aa1cfcbf3f8\",\n      \"name\": \"e06ebcd7-3269-46a4-b719-7aa1cfcbf3f8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T17:35:51.643483493Z\",\n      \"endTime\": \"2026-09-21T17:59:51.42203769Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svh48rd484rghmkk5f5u7tahpl40sqm still exists (deletion dispatched at 2026-09-21T17:35:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a0e198\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-9vcphdzd6v79\" to finish deleting: GET https://rp.j9521664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e06ebcd7-3269-46a4-b719-7aa1cfcbf3f8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e06ebcd7-3269-46a4-b719-7aa1cfcbf3f8\",\n      \"name\": \"e06ebcd7-3269-46a4-b719-7aa1cfcbf3f8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T17:35:51.643483493Z\",\n      \"endTime\": \"2026-09-21T17:59:51.42203769Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svh48rd484rghmkk5f5u7tahpl40sqm still exists (deletion dispatched at 2026-09-21T17:35:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102066961654484992","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6577,"pr_state":"open","pr_sha":"edba6d9d360500528c47783549af6c9d12663742","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T15:43:16Z","started_at":"2026-09-21T15:43:16Z","completed_at":"2026-09-21T18:14:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T18:15:55.648174124Z","timing_metadata_checked_at":"2026-09-21T18:15:55.648174124Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T18:15:55.651342814Z","region_metadata_checked_at":"2026-09-21T18:15:55.651342814Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"9ecd3678cd4fa26153cb39ec634431738e593b5c65446b76116e6d52b531b255","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102066961654484992","occurred_at":"2026-09-21T15:43:16Z","failed_at":"alert","signature_id":"cf976378f58d07a1907e42920194c35d042c96ceb85f1605def2a3e4f31965b7","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-21T18:00:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j4484992-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-vcmjr\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-vcmjr (namespace arobit) on cluster ci01-j4484992-svc has averaged 125.6% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"a5e120dee5673987c76439cd7f79567deba3c9688c1ef92a02429c086f45d3b5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102066961654484992","occurred_at":"2026-09-21T15:43:16Z","failed_at":"e2e","signature_id":"3baf717c6e038e95a12a4914ecbfc7314ded673acef6f6d69b55d6ffd81cf2c3","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.006s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001389008\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.22.13) failed: no version in clusterversion/version status.history is greater than initial \"4.22.13\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.006s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001389008\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.22.13) failed: no version in clusterversion/version status.history is greater than initial \"4.22.13\"\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ccf8ebc7f99dd3758df5a50af86c29fba7755ee8f0d641da7630a416917e5d64","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4"}},{"row_id":"ec7f17c133a26a9985e05a190d2c149baf5fbb0d36225244efaa8f11c8cbcd73","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102066961654484992","occurred_at":"2026-09-21T15:43:16Z","failed_at":"alert","signature_id":"1e2a2d6ea9e82c8b335238980344ef522bc23404f4fd03c581bdabd25d88347a","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-21T17:00:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j4484992-svc\", component=\"backend\", name=\"controlplaneversionstatuscollector\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue controlplaneversionstatuscollector has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired"}},{"row_id":"f6ac5817f99b157c4be9f25cf4cbc393d3805527e5d8c007b806df93562c718d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102066961654484992","occurred_at":"2026-09-21T15:43:16Z","failed_at":"e2e","signature_id":"c3a5e1465459779327c6aeb29818e28c228d277e9e34a3ff31c5444512e3306b","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000c984b0\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.33) failed: no version in clusterversion/version status.history is greater than initial \"4.21.33\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000c984b0\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.33) failed: no version in clusterversion/version status.history is greater than initial \"4.21.33\"\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ccf8ebc7f99dd3758df5a50af86c29fba7755ee8f0d641da7630a416917e5d64","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","ccf8ebc7f99dd3758df5a50af86c29fba7755ee8f0d641da7630a416917e5d64","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6891/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102071948841324544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6891,"pr_state":"closed","pr_sha":"d8bb7efc092c2e7d5ef7b195e7d9015c5f422a90","final_merged_sha":"d8bb7efc092c2e7d5ef7b195e7d9015c5f422a90","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T15:22:58Z","started_at":"2026-09-21T15:22:58Z","completed_at":"2026-09-21T18:58:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T19:09:08.357040952Z","timing_metadata_checked_at":"2026-09-21T19:09:08.357040952Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T19:09:08.359661244Z","region_metadata_checked_at":"2026-09-21T19:09:08.359661244Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"7ddaa566371435c8c67e9082c13f6fe821d0315772b50416c9cf176010ec3477","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6891/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102071948841324544","occurred_at":"2026-09-21T15:22:58Z","failed_at":"alert","signature_id":"574e2d37afd01dd117c49baab3fa41bc89b333d6a88fa5a2f0f78db84b23d7aa","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T18:10:16Z\n  Ended:       2026-09-21T18:24:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j1324544-svc\", component=\"frontend\", method=\"delete\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: More than 5% of frontend requests for delete /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j1324544-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}},{"row_id":"9a673e4fa8191bafaf43ea4ad8ac978306f08c41a8f8b5676d2f75092cf95ca5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6891/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102071948841324544","occurred_at":"2026-09-21T15:22:58Z","failed_at":"e2e","signature_id":"91c6da1e9449757d29ef7e57923f4eca5b5ac2181ece3591996f025be6912e6d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-pkrdhh\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005f0960\u003e: \n    DNS for route host agnhost-e2e-sample-app-zhfc5.apps.aro.e2n6l9t1x7s8m0x.srbh.j1324544.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-zhfc5.apps.aro.e2n6l9t1x7s8m0x.srbh.j1324544.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-zhfc5.apps.aro.e2n6l9t1x7s8m0x.srbh.j1324544.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-pkrdhh\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005f0960\u003e: \n    DNS for route host agnhost-e2e-sample-app-zhfc5.apps.aro.e2n6l9t1x7s8m0x.srbh.j1324544.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-zhfc5.apps.aro.e2n6l9t1x7s8m0x.srbh.j1324544.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-zhfc5.apps.aro.e2n6l9t1x7s8m0x.srbh.j1324544.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-zhfc5.apps.aro.e2n6l9t1x7s8m0x.srbh.j1324544.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-zhfc5.apps.aro.e2n6l9t1x7s8m0x.srbh.j1324544.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-zhfc5.apps.aro.e2n6l9t1x7s8m0x.srbh.j1324544.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded"}},{"row_id":"d2e69f9cf7ccf5147539e0aa6cfec3f4edd78db30e06f10103ea0ff15dce08c7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6891/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102071948841324544","occurred_at":"2026-09-21T15:22:58Z","failed_at":"e2e","signature_id":"a9087b3ae48f305774221ee6543503f7dfc65fcd36dfb5b29c8030232752001a","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001666180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-wqw7tdjdvp76\" to finish deleting: GET https://rp.j1324544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a334ae8-bd7b-4804-bb8e-e36f4a501d5b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a334ae8-bd7b-4804-bb8e-e36f4a501d5b\",\n      \"name\": \"5a334ae8-bd7b-4804-bb8e-e36f4a501d5b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T18:16:30.094041656Z\",\n      \"endTime\": \"2026-09-21T18:40:38.008271865Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svhme1p7pvrn3f8ccifc33jt21fnfd1 still exists (deletion dispatched at 2026-09-21T18:16:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001666180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-wqw7tdjdvp76\" to finish deleting: GET https://rp.j1324544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a334ae8-bd7b-4804-bb8e-e36f4a501d5b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a334ae8-bd7b-4804-bb8e-e36f4a501d5b\",\n      \"name\": \"5a334ae8-bd7b-4804-bb8e-e36f4a501d5b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T18:16:30.094041656Z\",\n      \"endTime\": \"2026-09-21T18:40:38.008271865Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svhme1p7pvrn3f8ccifc33jt21fnfd1 still exists (deletion dispatched at 2026-09-21T18:16:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d67ab5e7a363769f9517f183ebc78bbacdc6d23e44c84d6bd607f543316c129a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6891/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102071948841324544","occurred_at":"2026-09-21T15:22:58Z","failed_at":"alert","signature_id":"86bfa7526a8d7d5c41009ddf1e0f82889a3421685de0e2689447600cbba9ed74","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T17:36:17Z\n  Ended:       2026-09-21T18:10:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1324544-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1324544-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102073111963766784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6965,"pr_state":"closed","pr_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","final_merged_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T15:07:35Z","started_at":"2026-09-21T15:07:35Z","completed_at":"2026-09-21T18:49:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T18:58:50.553034636Z","timing_metadata_checked_at":"2026-09-21T18:58:50.553034636Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T18:58:50.556925624Z","region_metadata_checked_at":"2026-09-21T18:58:50.556925624Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"b122af4eb4f88f3a999c9de09ce0b798b5a2ec7d1516f2220c9ac1a6dcfd3711","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102073111963766784","occurred_at":"2026-09-21T15:07:35Z","failed_at":"e2e","signature_id":"f9a1d285355f6abfe6a2500869cc6754a5128898c888d0876470311c233867d2","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000914300\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-27x5trtjwxsk\" to finish deleting: GET https://rp.j3766784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6e29c50f-5305-4d4e-9d1b-1baf6cc3d053\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6e29c50f-5305-4d4e-9d1b-1baf6cc3d053\",\n      \"name\": \"6e29c50f-5305-4d4e-9d1b-1baf6cc3d053\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T18:03:50.796470355Z\",\n      \"endTime\": \"2026-09-21T18:27:50.905717326Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svhin5lqriup0dr3kjce65k83gu18lc still exists (deletion dispatched at 2026-09-21T18:03:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000914300\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-27x5trtjwxsk\" to finish deleting: GET https://rp.j3766784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6e29c50f-5305-4d4e-9d1b-1baf6cc3d053\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6e29c50f-5305-4d4e-9d1b-1baf6cc3d053\",\n      \"name\": \"6e29c50f-5305-4d4e-9d1b-1baf6cc3d053\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T18:03:50.796470355Z\",\n      \"endTime\": \"2026-09-21T18:27:50.905717326Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svhin5lqriup0dr3kjce65k83gu18lc still exists (deletion dispatched at 2026-09-21T18:03:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"edfe3fd1756ebd5a53c07435c349d025a1d5ef263880fe085eacc7b5697ea0f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102073111963766784","occurred_at":"2026-09-21T15:07:35Z","failed_at":"alert","signature_id":"6621430e792a67ddd0ea4ee5d1f680b8452a866fb2c92438b1cebc8c139f2f03","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T18:05:55Z\n  Ended:       2026-09-21T18:11:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j3766784-svc\", component=\"frontend\", method=\"get\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j3766784-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102058858829582336","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6886,"pr_state":"closed","pr_sha":"a6aa6a03a137ec55e3f4f5d06cc620e6bf6dd7f3","final_merged_sha":"a6aa6a03a137ec55e3f4f5d06cc620e6bf6dd7f3","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T15:00:39Z","started_at":"2026-09-21T15:00:39Z","completed_at":"2026-09-21T17:48:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T17:53:59.751648889Z","timing_metadata_checked_at":"2026-09-21T17:53:59.751648889Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T17:53:59.754774179Z","region_metadata_checked_at":"2026-09-21T17:53:59.754774179Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"daf341feca30015402ee8b36e661527a9fbb3082bce0ef5a8a4c261746c41d85","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102058858829582336","occurred_at":"2026-09-21T15:00:39Z","failed_at":"e2e","signature_id":"ec02407a8e4b2bba534233c5100821ad7063c3e19b6b4a6eea807151b4e54aef","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019e20c0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-pmx6cfl658sr\" to finish deleting: GET https://rp.j9582336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b19e568c-3189-499b-bcc0-805dcde97620\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b19e568c-3189-499b-bcc0-805dcde97620\",\n      \"name\": \"b19e568c-3189-499b-bcc0-805dcde97620\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T17:06:30.456970874Z\",\n      \"endTime\": \"2026-09-21T17:30:35.786842116Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svgm02gdmhm41sp2jggubv9mppisfs3 still exists (deletion dispatched at 2026-09-21T17:06:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019e20c0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-pmx6cfl658sr\" to finish deleting: GET https://rp.j9582336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b19e568c-3189-499b-bcc0-805dcde97620\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b19e568c-3189-499b-bcc0-805dcde97620\",\n      \"name\": \"b19e568c-3189-499b-bcc0-805dcde97620\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T17:06:30.456970874Z\",\n      \"endTime\": \"2026-09-21T17:30:35.786842116Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svgm02gdmhm41sp2jggubv9mppisfs3 still exists (deletion dispatched at 2026-09-21T17:06:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102052987269550080","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6995,"pr_state":"closed","pr_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","final_merged_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T15:00:31Z","started_at":"2026-09-21T15:00:31Z","completed_at":"2026-09-21T17:16:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T17:22:32.722086494Z","timing_metadata_checked_at":"2026-09-21T17:22:32.722086494Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T17:22:32.725053884Z","region_metadata_checked_at":"2026-09-21T17:22:32.725053884Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"42880544d0f18ca87640e643b8fadd492e6bdc75dc38814e64c3b40fcbee064d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102052987269550080","occurred_at":"2026-09-21T15:00:31Z","failed_at":"e2e","signature_id":"68b07905d42fa8e6eec867687205ede718fc89fec8d45072784982746b973a77","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.38\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b37240\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-gvbn8z\" in resourcegroup=\"rg-np-version-upgrade-gvbn8z-hkxf5kh952mz\" to finish creating: GET https://rp.j9550080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34cf9dc9-964d-48aa-a9fe-30aa7b3ae5d7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34cf9dc9-964d-48aa-a9fe-30aa7b3ae5d7\",\n      \"name\": \"34cf9dc9-964d-48aa-a9fe-30aa7b3ae5d7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T16:16:27.808191119Z\",\n      \"endTime\": \"2026-09-21T16:35:35.728389671Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.38\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b37240\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-gvbn8z\" in resourcegroup=\"rg-np-version-upgrade-gvbn8z-hkxf5kh952mz\" to finish creating: GET https://rp.j9550080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34cf9dc9-964d-48aa-a9fe-30aa7b3ae5d7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34cf9dc9-964d-48aa-a9fe-30aa7b3ae5d7\",\n      \"name\": \"34cf9dc9-964d-48aa-a9fe-30aa7b3ae5d7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T16:16:27.808191119Z\",\n      \"endTime\": \"2026-09-21T16:35:35.728389671Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7031/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102066654929227776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7031,"pr_state":"open","pr_sha":"a6af8a6f4fa5f4a84a4cea1ddea63e410d5cd928","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T14:59:52Z","started_at":"2026-09-21T14:59:52Z","completed_at":"2026-09-21T18:12:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T18:15:55.863629247Z","timing_metadata_checked_at":"2026-09-21T18:15:55.863629247Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T18:15:55.867089636Z","region_metadata_checked_at":"2026-09-21T18:15:55.867089636Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"4fa003f4a0dcd4c239c0167095a12074a536966540ec0078d152cd40ba4552da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7031/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102066654929227776","occurred_at":"2026-09-21T14:59:52Z","failed_at":"e2e","signature_id":"89ef15c875916c7039f548b354d242edd5d8d23df48a86beaadfbd6cfa6da6ce","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:293]: failed to create cluster role binding for external auth via VM\nUnexpected error:\n    \u003c*errors.errorString | 0xc000aafea0\u003e: \n    error: failed to create clusterrolebinding: etcdserver: request timed out, possibly due to previous leader failure\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:293]: failed to create cluster role binding for external auth via VM\nUnexpected error:\n    \u003c*errors.errorString | 0xc000aafea0\u003e: \n    error: failed to create clusterrolebinding: etcdserver: request timed out, possibly due to previous leader failure\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"68c7b4b187574b9d0d321c3db18bc4ea80329f6d4b3ee8c01eea35e77ac63cfb","failure_pattern":"error: failed to create clusterrolebinding: etcdserver: request timed out, possibly due to previous leader failure","search_query":"error: failed to create clusterrolebinding: etcdserver: request timed out, possibly due to previous leader failure"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["68c7b4b187574b9d0d321c3db18bc4ea80329f6d4b3ee8c01eea35e77ac63cfb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102079066994642944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7044,"pr_state":"closed","pr_sha":"0f625957a86b5fda4999ce65087e506bc2499c89","final_merged_sha":"0f625957a86b5fda4999ce65087e506bc2499c89","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T14:23:45Z","started_at":"2026-09-21T14:23:45Z","completed_at":"2026-09-21T17:45:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T17:54:00.162126481Z","timing_metadata_checked_at":"2026-09-21T17:54:00.162126481Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T17:54:00.165118771Z","region_metadata_checked_at":"2026-09-21T17:54:00.165118771Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"8f4907c0f3c8951e9c4db0a74e183bd7321e73a3a9a330baca0c94b86171b41d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102079066994642944","occurred_at":"2026-09-21T14:23:45Z","failed_at":"provision","signature_id":"19498df56e93c37fe7f6fb08ad99bc2b1b1c8d05100b0a59886217e4fe1408e0","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/storageclass (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-21T17:32:18.785Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 description=\"Step storageclass\\n Kind: Helm\\n\"\ntime=2026-09-21T17:32:19.291Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 values=\"map[skuName:PremiumV2_LRS unannotateJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23]\"\ntime=2026-09-21T17:32:19.291Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-09-21T17:32:19.604Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 namespace=default-sc\ntime=2026-09-21T17:32:19.952Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 dryRun=false\ntime=2026-09-21T17:32:19.952Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-09-21T17:32:20.082Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-09-21T17:37:24.746Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 err=\"failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded\"\ntime=2026-09-21T17:37:24.746Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"abcd5f9b62e8cf7d13105a0c9ded33446cbd1e1f876e962101b7bbf9a2c604c6","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: InProgress, message: Job in progress context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["abcd5f9b62e8cf7d13105a0c9ded33446cbd1e1f876e962101b7bbf9a2c604c6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102036202222784512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6965,"pr_state":"closed","pr_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","final_merged_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T14:00:43Z","started_at":"2026-09-21T14:00:43Z","completed_at":"2026-09-21T14:38:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T14:54:41.653427612Z","timing_metadata_checked_at":"2026-09-21T14:54:41.653427612Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T14:54:41.657482899Z","region_metadata_checked_at":"2026-09-21T14:54:41.657482899Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"fb154cd3e0e29890b16d12be503d49d5078bd2da2108b47c9e7442f086aea832","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102036202222784512","occurred_at":"2026-09-21T14:00:43Z","failed_at":"provision","signature_id":"c31c6107cbf555f3a1c1e24740435c96d16858d6b017c1f643029792ec2069ee","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/admin-api-cert","test_suite":"templatize-pipeline","failure_text":"time=2026-09-21T14:33:07.930Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert description=\"Step admin-api-cert\\n Kind: CreateCertificate\\n\"\ntime=2026-09-21T14:33:07.930Z level=INFO msg=\"Creating certificate\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert vaultBaseUrl=https://aro-hcp-dev-svc-kv.vault.azure.net/ certificateName=admin-api-cert-ci01-j2784512 issuer=Self commonName=admin.j2784512.hcpsvc.osadev.cloud san=admin.j2784512.hcpsvc.osadev.cloud\ntime=2026-09-21T14:33:08.799Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert err=\"error running Create Certificate Step: failed to create certificate \\\"admin-api-cert-ci01-j2784512\\\" in vault \\\"https://aro-hcp-dev-svc-kv.vault.azure.net/\\\": POST https://aro-hcp-dev-svc-kv.vault.azure.net/certificates/admin-api-cert-ci01-j2784512/create\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Certificate admin-api-cert-ci01-j2784512 is currently in a deleted but recoverable state, and its name cannot be reused; in this state, the certificate can only be recovered or purged.\\\",\\n    \\\"innererror\\\": {\\n      \\\"code\\\": \\\"ObjectIsDeletedButRecoverable\\\"\\n    }\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6f6e7e66ad8bc42bd6647f1e246b1e77205441e0c9495974d9b1f940d8bdec9c","failure_pattern":"ERROR CODE: Conflict; detail code ObjectIsDeletedButRecoverable","search_query":"ERROR CODE: Conflict"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6f6e7e66ad8bc42bd6647f1e246b1e77205441e0c9495974d9b1f940d8bdec9c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102025876047663104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7001,"pr_state":"open","pr_sha":"293afd22cf50b4ca536635d9a7c22971120e5a9b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T13:23:17Z","started_at":"2026-09-21T13:23:17Z","completed_at":"2026-09-21T15:37:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T15:57:52.142111409Z","timing_metadata_checked_at":"2026-09-21T15:57:52.142111409Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T15:57:52.145200598Z","region_metadata_checked_at":"2026-09-21T15:57:52.145200598Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"0db5188b9aa6e30ce3c7d2c576985ab080ddfd51ac87fe9934d875ab10766e38","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102025876047663104","occurred_at":"2026-09-21T13:23:17Z","failed_at":"e2e","signature_id":"7a9f86ce8e43b028b3087e8aed9b7165fa70c0b83864671666f6c09e2219b3bf","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001142180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-d928sfgs7x2n\" to finish deleting: GET https://rp.j7663104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8a5ddcdd-2691-41a8-bda5-ee1dbda305c5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8a5ddcdd-2691-41a8-bda5-ee1dbda305c5\",\n      \"name\": \"8a5ddcdd-2691-41a8-bda5-ee1dbda305c5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T14:57:37.927640106Z\",\n      \"endTime\": \"2026-09-21T15:21:41.429457146Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svepuc5qf5lfhoj9ohftvovivfoa3vb still exists (deletion dispatched at 2026-09-21T14:57:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001142180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-d928sfgs7x2n\" to finish deleting: GET https://rp.j7663104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8a5ddcdd-2691-41a8-bda5-ee1dbda305c5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8a5ddcdd-2691-41a8-bda5-ee1dbda305c5\",\n      \"name\": \"8a5ddcdd-2691-41a8-bda5-ee1dbda305c5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T14:57:37.927640106Z\",\n      \"endTime\": \"2026-09-21T15:21:41.429457146Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svepuc5qf5lfhoj9ohftvovivfoa3vb still exists (deletion dispatched at 2026-09-21T14:57:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102025100818649088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7009,"pr_state":"closed","pr_sha":"3ea999746a8bb5c1666c5bd5b4678a5c293d1c76","final_merged_sha":"c83bdabaf08bdb32938b4526f3e12a94c26f811c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T13:20:12Z","started_at":"2026-09-21T13:20:12Z","completed_at":"2026-09-21T15:27:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T15:47:13.074471618Z","timing_metadata_checked_at":"2026-09-21T15:47:13.074471618Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T15:47:13.077048209Z","region_metadata_checked_at":"2026-09-21T15:47:13.077048209Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"160e302ef3b091b343f5dcc086393dc4e9ce51dcc82572e7bea4e76a0e40b528","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102025100818649088","occurred_at":"2026-09-21T13:20:12Z","failed_at":"e2e","signature_id":"eb38a247e7c4c687bafc7635602cd03043541f77a3b2136084601a1dc061ce6e","test_name":"KSM HCP Metrics kube_node_info metrics should be present in Azure Monitor for the happy-path cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:91]: Timed out after 900.001s.\nkube_node_info metrics never appeared in Azure Monitor for any hostedcontrolplane\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:89 with:\nexpected kube_node_info metrics for at least one hostedcontrolplane but got no results\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:91]: Timed out after 900.001s.\nkube_node_info metrics never appeared in Azure Monitor for any hostedcontrolplane\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:89 with:\nexpected kube_node_info metrics for at least one hostedcontrolplane but got no results\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"35fcc68c27835aae14daafdef12fc813fb2e38cc3e35d2f974eb6cc333b3ce7e","failure_pattern":"kube_node_info metrics never appeared in Azure Monitor for any hostedcontrolplane","search_query":"kube_node_info metrics never appeared in Azure Monitor for any hostedcontrolplane"}},{"row_id":"a7c0dd332a44a11920dc563039aedc4ebc47f0bc88576641eb807fdfeb778670","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102025100818649088","occurred_at":"2026-09-21T13:20:12Z","failed_at":"e2e","signature_id":"e5817129d14143b2f0d37d6a9804a3b5dfa2dcb622e117ee3199442dccb64bfb","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019ba078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-xqhvgr\" in resourcegroup=\"rg-zstream-upgrade-4-21-w7f2d6mqncj6\" to finish deleting: GET https://rp.j8649088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d6854548-744e-4bc9-8180-91ed27ffe894\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d6854548-744e-4bc9-8180-91ed27ffe894\",\n      \"name\": \"d6854548-744e-4bc9-8180-91ed27ffe894\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T14:41:36.255977401Z\",\n      \"endTime\": \"2026-09-21T15:05:36.956425706Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sveqe4p0n12d23lj6lki2mdamppdul4 still exists (deletion dispatched at 2026-09-21T14:41:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019ba078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-xqhvgr\" in resourcegroup=\"rg-zstream-upgrade-4-21-w7f2d6mqncj6\" to finish deleting: GET https://rp.j8649088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d6854548-744e-4bc9-8180-91ed27ffe894\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d6854548-744e-4bc9-8180-91ed27ffe894\",\n      \"name\": \"d6854548-744e-4bc9-8180-91ed27ffe894\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T14:41:36.255977401Z\",\n      \"endTime\": \"2026-09-21T15:05:36.956425706Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sveqe4p0n12d23lj6lki2mdamppdul4 still exists (deletion dispatched at 2026-09-21T14:41:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["35fcc68c27835aae14daafdef12fc813fb2e38cc3e35d2f974eb6cc333b3ce7e","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102020232922009600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T13:00:43Z","started_at":"2026-09-21T13:00:43Z","completed_at":"2026-09-21T15:35:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T15:47:12.887589542Z","timing_metadata_checked_at":"2026-09-21T15:47:12.887589542Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T15:47:12.890873831Z","region_metadata_checked_at":"2026-09-21T15:47:12.890873831Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"1979372fcf1fc0133d1e7c0436bd474f5bd8fccc0f7cdf43140b9b8c58c714a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102020232922009600","occurred_at":"2026-09-21T13:00:43Z","failed_at":"e2e","signature_id":"f98417bb9cbfe1c0a40caa22135205b1146d1848d0521a818729663395e61826","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000bd6318\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-lng75bvlnlb2\" to finish deleting: GET https://rp.j2009600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f7a28b-d0c3-43fd-bf4a-f720601b1392\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f7a28b-d0c3-43fd-bf4a-f720601b1392\",\n      \"name\": \"90f7a28b-d0c3-43fd-bf4a-f720601b1392\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T14:52:42.031743431Z\",\n      \"endTime\": \"2026-09-21T15:16:43.14561436Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svemd9v3mhmh6seeg6en1gro78cs5it still exists (deletion dispatched at 2026-09-21T14:52:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000bd6318\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-lng75bvlnlb2\" to finish deleting: GET https://rp.j2009600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f7a28b-d0c3-43fd-bf4a-f720601b1392\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f7a28b-d0c3-43fd-bf4a-f720601b1392\",\n      \"name\": \"90f7a28b-d0c3-43fd-bf4a-f720601b1392\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T14:52:42.031743431Z\",\n      \"endTime\": \"2026-09-21T15:16:43.14561436Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svemd9v3mhmh6seeg6en1gro78cs5it still exists (deletion dispatched at 2026-09-21T14:52:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cccaffe2060dbf881b0cfaa8bd6863243537ccfc86ebfd651d35ef7b8ccfc5b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102020232922009600","occurred_at":"2026-09-21T13:00:43Z","failed_at":"e2e","signature_id":"27e0115d202f8dcaf0a7b6feb4e03a4f922d1335c323dc875e801cae30dafda9","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010c40d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qjglkd97dxdj\" to finish deleting: GET https://rp.j2009600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/215dd062-958f-48e1-ad5e-06c604764df4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/215dd062-958f-48e1-ad5e-06c604764df4\",\n      \"name\": \"215dd062-958f-48e1-ad5e-06c604764df4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T14:32:19.890700893Z\",\n      \"endTime\": \"2026-09-21T14:56:24.375099539Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svemfek67g4thevlao08ncrlk8iuckk still exists (deletion dispatched at 2026-09-21T14:32:21Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010c40d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qjglkd97dxdj\" to finish deleting: GET https://rp.j2009600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/215dd062-958f-48e1-ad5e-06c604764df4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/215dd062-958f-48e1-ad5e-06c604764df4\",\n      \"name\": \"215dd062-958f-48e1-ad5e-06c604764df4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T14:32:19.890700893Z\",\n      \"endTime\": \"2026-09-21T14:56:24.375099539Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svemfek67g4thevlao08ncrlk8iuckk still exists (deletion dispatched at 2026-09-21T14:32:21Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102020127040999424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7033,"pr_state":"closed","pr_sha":"c448a0db20d77ea047c00534044b12cc2b2ac565","final_merged_sha":"ef821496d20065a3db001282a41d83827d994051","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T13:00:26Z","started_at":"2026-09-21T13:00:26Z","completed_at":"2026-09-21T15:34:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T15:47:13.255522914Z","timing_metadata_checked_at":"2026-09-21T15:47:13.255522914Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T15:47:13.258145205Z","region_metadata_checked_at":"2026-09-21T15:47:13.258145205Z"},"failed_at":["alert","e2e"],"failed_test_count":8,"occurrences":[{"row_id":"3bbc94acdd785bddd90bb3552cd556e35ae67a0de808c5f17ac28c1d2485dba8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102020127040999424","occurred_at":"2026-09-21T13:00:26Z","failed_at":"e2e","signature_id":"f13ebee297ee756079858e27e2215fdf9f11d8fe7cc1bd7d1b8c14e2a1791a19","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b68880\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-t9sfgdtzmdz6\" to finish creating: GET https://rp.j0999424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a83df19f-3103-42e8-8de4-e77b74ab884a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a83df19f-3103-42e8-8de4-e77b74ab884a\",\n      \"name\": \"a83df19f-3103-42e8-8de4-e77b74ab884a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T14:10:33.153956692Z\",\n      \"endTime\": \"2026-09-21T14:39:36.608452036Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b68880\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-t9sfgdtzmdz6\" to finish creating: GET https://rp.j0999424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a83df19f-3103-42e8-8de4-e77b74ab884a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a83df19f-3103-42e8-8de4-e77b74ab884a\",\n      \"name\": \"a83df19f-3103-42e8-8de4-e77b74ab884a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T14:10:33.153956692Z\",\n      \"endTime\": \"2026-09-21T14:39:36.608452036Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4595b333c67dd33f68dc207f648131b207e243d58d87d73216de5025ae221c90","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102020127040999424","occurred_at":"2026-09-21T13:00:26Z","failed_at":"e2e","signature_id":"14f4cbe167bfd8c287677e3dc90213baa6119f187877464394153c144c4899b5","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:85]: failed to create customer resources for external auth cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00072ec40\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-ea-cluster-sntc52\" in resourcegroup=\"external-auth-cluster-bkt2n5ldrfvt\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-bkt2n5ldrfvt/providers/Microsoft.Resources/deployments/customer-infra-ea-cluster-sntc52/operationStatuses/08584116072201723846\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n    \\\"message\\\": \\\"The vault name 'cust-kv-xkv3iaodnbzd2' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:85]: failed to create customer resources for external auth cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00072ec40\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-ea-cluster-sntc52\" in resourcegroup=\"external-auth-cluster-bkt2n5ldrfvt\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-bkt2n5ldrfvt/providers/Microsoft.Resources/deployments/customer-infra-ea-cluster-sntc52/operationStatuses/08584116072201723846\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n    \\\"message\\\": \\\"The vault name 'cust-kv-xkv3iaodnbzd2' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0d7a8e5dc2d8e7103fa834285288a1f3b3954abe71b752dee4412a0b1b6737d5","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"55584726a70c34e8de7bdb4686e5318b22952362535a6df8df8181c65a4c0a2f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102020127040999424","occurred_at":"2026-09-21T13:00:26Z","failed_at":"e2e","signature_id":"34e18bcbecd4aaa5ee867c10ac4f9c921b095d8c5be502571efbfcabee3107e5","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001966090\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-rhjts8hwrjvd\" to finish deleting: GET https://rp.j0999424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d9c34ef0-8a09-421e-a4c8-24295ff4827b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d9c34ef0-8a09-421e-a4c8-24295ff4827b\",\n      \"name\": \"d9c34ef0-8a09-421e-a4c8-24295ff4827b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T14:53:48.99729819Z\",\n      \"endTime\": \"2026-09-21T15:17:48.099903279Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sveoskujsiok1rvvgaa3rtbbvldoc3f still exists (deletion dispatched at 2026-09-21T14:53:51Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001966090\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-rhjts8hwrjvd\" to finish deleting: GET https://rp.j0999424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d9c34ef0-8a09-421e-a4c8-24295ff4827b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d9c34ef0-8a09-421e-a4c8-24295ff4827b\",\n      \"name\": \"d9c34ef0-8a09-421e-a4c8-24295ff4827b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T14:53:48.99729819Z\",\n      \"endTime\": \"2026-09-21T15:17:48.099903279Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sveoskujsiok1rvvgaa3rtbbvldoc3f still exists (deletion dispatched at 2026-09-21T14:53:51Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6adca7182017abac7b3ec06450ccaf5bb944b043ff6b4593b14daf668fa4402b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102020127040999424","occurred_at":"2026-09-21T13:00:26Z","failed_at":"alert","signature_id":"f6904acfac7e72c3440aef5f7b913bc8953662014ff3e533f37d114c8df2ab51","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T14:32:16Z\n  Ended:       2026-09-21T14:58:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0999424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0999424/providers/microsoft.monitor/accounts/hcps-j0999424\", namespace=\"ocm-arohcpci01-2svepmb9pe39k53ku2oiti573ipu6mda-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-mbr7z\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2svepmb9pe39k53ku2oiti573ipu6mda-cilium-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-21T14:32:16Z\n  Ended:       2026-09-21T15:00:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0999424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0999424/providers/microsoft.monitor/accounts/hcps-j0999424\", namespace=\"ocm-arohcpci01-2svepmb9pe39k53ku2oiti573ipu6mda-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-mbr7z\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2svepmb9pe39k53ku2oiti573ipu6mda-cilium-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"9bf0a4b37e66dcabb2992a2fdebb605827e29cee1642f94d2c897aca290ab018","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102020127040999424","occurred_at":"2026-09-21T13:00:26Z","failed_at":"e2e","signature_id":"89c279997a7f7ba146d4cc5af783051974c52ed84265d8068710d18560ba8f57","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:116]: failed to verify simple web app runs on cluster \"cluster-hs-pre-8gb9mx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011931e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-28d65.apps.aro.o4p4d0n2f5l6i5t.cztj.j0999424.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-28d65.apps.aro.o4p4d0n2f5l6i5t.cztj.j0999424.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-28d65.apps.aro.o4p4d0n2f5l6i5t.cztj.j0999424.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:116]: failed to verify simple web app runs on cluster \"cluster-hs-pre-8gb9mx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011931e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-28d65.apps.aro.o4p4d0n2f5l6i5t.cztj.j0999424.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-28d65.apps.aro.o4p4d0n2f5l6i5t.cztj.j0999424.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-28d65.apps.aro.o4p4d0n2f5l6i5t.cztj.j0999424.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-28d65.apps.aro.o4p4d0n2f5l6i5t.cztj.j0999424.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-28d65.apps.aro.o4p4d0n2f5l6i5t.cztj.j0999424.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-28d65.apps.aro.o4p4d0n2f5l6i5t.cztj.j0999424.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"a113877286604fc179e2f590f7f47af7f75ce47d052db37b3658740a75c9c7bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102020127040999424","occurred_at":"2026-09-21T13:00:26Z","failed_at":"e2e","signature_id":"690754dd0c89156393ceb3dead1cd2a60e8d9c59d6c5ba2678c452e149963413","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014e60f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-5c54p2\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-zfjscslq4q9c\" to finish deleting: GET https://rp.j0999424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/df1c87e9-2c94-400b-8921-df6f8139f2ff\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/df1c87e9-2c94-400b-8921-df6f8139f2ff\",\n      \"name\": \"df1c87e9-2c94-400b-8921-df6f8139f2ff\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T14:41:02.069962942Z\",\n      \"endTime\": \"2026-09-21T15:05:06.609792809Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sveov47cucnca794m565e9cofa12jsp still exists (deletion dispatched at 2026-09-21T14:41:04Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014e60f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-5c54p2\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-zfjscslq4q9c\" to finish deleting: GET https://rp.j0999424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/df1c87e9-2c94-400b-8921-df6f8139f2ff\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/df1c87e9-2c94-400b-8921-df6f8139f2ff\",\n      \"name\": \"df1c87e9-2c94-400b-8921-df6f8139f2ff\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T14:41:02.069962942Z\",\n      \"endTime\": \"2026-09-21T15:05:06.609792809Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sveov47cucnca794m565e9cofa12jsp still exists (deletion dispatched at 2026-09-21T14:41:04Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cddeea9888c9d89db53725dbd975a2a15eb58ff5fe9b034bb94b6760b741d880","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102020127040999424","occurred_at":"2026-09-21T13:00:26Z","failed_at":"alert","signature_id":"2d3718d83c441f59e92416d821f7910b3e379267b611456fdcb9b69e5c823fe2","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T14:46:09Z\n  Ended:       2026-09-21T14:58:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0999424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0999424/providers/microsoft.monitor/accounts/hcps-j0999424\", namespace=\"ocm-arohcpci01-2svepmb9pe39k53ku2oiti573ipu6mda-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-mbr7z\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2svepmb9pe39k53ku2oiti573ipu6mda-cilium-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"ed92e5ae9a116802231281417e963bf2be703e345253e321e511ac99d9bb06ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102020127040999424","occurred_at":"2026-09-21T13:00:26Z","failed_at":"alert","signature_id":"807d1b3879be556e88cc30f4a64c46996629f8ac6923d5ba5c4fcade28823d90","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T14:45:59Z\n  Ended:       2026-09-21T14:51:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0999424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0999424/providers/microsoft.monitor/accounts/services-j0999424\", namespace=\"klusterlet-2svepmb9pe39k53ku2oiti573ipu6mda\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-mbr7z\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2svepmb9pe39k53ku2oiti573ipu6mda/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-21T14:45:59Z\n  Ended:       2026-09-21T14:51:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0999424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0999424/providers/microsoft.monitor/accounts/services-j0999424\", namespace=\"klusterlet-2svepmb9pe39k53ku2oiti573ipu6mda\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-mbr7z\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2svepmb9pe39k53ku2oiti573ipu6mda/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["0d7a8e5dc2d8e7103fa834285288a1f3b3954abe71b752dee4412a0b1b6737d5","13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102005383924027392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6886,"pr_state":"closed","pr_sha":"a6aa6a03a137ec55e3f4f5d06cc620e6bf6dd7f3","final_merged_sha":"a6aa6a03a137ec55e3f4f5d06cc620e6bf6dd7f3","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T12:01:02Z","started_at":"2026-09-21T12:01:02Z","completed_at":"2026-09-21T14:15:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T14:34:21.701526236Z","timing_metadata_checked_at":"2026-09-21T14:34:21.701526236Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T14:34:21.704809724Z","region_metadata_checked_at":"2026-09-21T14:34:21.704809724Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"6ef632776460263e5c81482d6088994cbe2ba4f2236672dddd73528466ef7234","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102005383924027392","occurred_at":"2026-09-21T12:01:02Z","failed_at":"e2e","signature_id":"bbb2a0cf39bb46b4833a734cf7e56dff14ace26e1c3dc69cb5fee77d8f3ba521","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010281e0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-pfpkf76vfcrs\" to finish deleting: GET https://rp.j4027392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a8c8fa1-89d6-4e11-9d1b-d0687fe50d94\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a8c8fa1-89d6-4e11-9d1b-d0687fe50d94\",\n      \"name\": \"5a8c8fa1-89d6-4e11-9d1b-d0687fe50d94\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T13:31:12.23375843Z\",\n      \"endTime\": \"2026-09-21T13:55:16.140974105Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svdie95fluircm3328v2lv3uu8t91u4 still exists (deletion dispatched at 2026-09-21T13:31:13Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010281e0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-pfpkf76vfcrs\" to finish deleting: GET https://rp.j4027392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a8c8fa1-89d6-4e11-9d1b-d0687fe50d94\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a8c8fa1-89d6-4e11-9d1b-d0687fe50d94\",\n      \"name\": \"5a8c8fa1-89d6-4e11-9d1b-d0687fe50d94\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T13:31:12.23375843Z\",\n      \"endTime\": \"2026-09-21T13:55:16.140974105Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svdie95fluircm3328v2lv3uu8t91u4 still exists (deletion dispatched at 2026-09-21T13:31:13Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102005345336430592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T12:00:59Z","started_at":"2026-09-21T12:00:59Z","completed_at":"2026-09-21T12:15:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T12:31:06.269986941Z","timing_metadata_checked_at":"2026-09-21T12:31:06.269986941Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T12:31:06.273772329Z","region_metadata_checked_at":"2026-09-21T12:31:06.273772329Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"8e9080970bbe8e37a36fa8dc029249a9ce3b43fe09e07cacd09eac38a116681b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102005345336430592","occurred_at":"2026-09-21T12:00:59Z","failed_at":"provision","signature_id":"09b1458b46da16205cd209ec402488a728d06d1e258b36845618ba3741f96692","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/infra-output","test_suite":"templatize-pipeline","failure_text":"time=2026-09-21T12:09:00.128Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output description=\"Step infra-output\\n  Kind: ARM\\n  Template: templates/svc-infra-lookup.bicep\\n  Parameters: configurations/svc-infra-lookup.tmpl.bicepparam\"\ntime=2026-09-21T12:09:00.546Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output\ntime=2026-09-21T12:09:01.165Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output deployment=44b6ca1d6f0df81628d0ba835badac62186b733c716b254d3b8b1c680dfe43f9 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6430592-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F44b6ca1d6f0df81628d0ba835badac62186b733c716b254d3b8b1c680dfe43f9\ntime=2026-09-21T12:09:31.310Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6430592-svc/providers/Microsoft.Resources/deployments/44b6ca1d6f0df81628d0ba835badac62186b733c716b254d3b8b1c680dfe43f9/operationStatuses/08584116143446905098\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"NotFound\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentNotFound\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Deployment 'svc-kv-xehipxbjxawki' could not be found.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aa573d68ede374d56fcd25dace5feffa8644835d49bf0c7ba52006e6de9139d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentNotFound; detail message Deployment 'svc-kv-xehipxbjxawki' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["8aa573d68ede374d56fcd25dace5feffa8644835d49bf0c7ba52006e6de9139d"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102005289971617792","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7044,"pr_state":"closed","pr_sha":"0f625957a86b5fda4999ce65087e506bc2499c89","final_merged_sha":"0f625957a86b5fda4999ce65087e506bc2499c89","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T12:00:55Z","started_at":"2026-09-21T12:00:55Z","completed_at":"2026-09-21T14:15:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T14:34:22.027048306Z","timing_metadata_checked_at":"2026-09-21T14:34:22.027048306Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T14:34:22.029512896Z","region_metadata_checked_at":"2026-09-21T14:34:22.029512896Z"},"failed_at":["alert","e2e"],"failed_test_count":8,"occurrences":[{"row_id":"60fba6686189ff570c2007cfff44623352bb0e1791e4a44ab0e28cc75730c5e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102005289971617792","occurred_at":"2026-09-21T12:00:55Z","failed_at":"e2e","signature_id":"3ef0cc1ba1412d2b9557e2a87b2d4d39bf6eb62cc2f335b2fffe273354eefae2","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001882150\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-bwv6gpm86sml\" to finish deleting: GET https://rp.j1617792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e63b539b-04f9-49eb-85fb-c1a75144fe9f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e63b539b-04f9-49eb-85fb-c1a75144fe9f\",\n      \"name\": \"e63b539b-04f9-49eb-85fb-c1a75144fe9f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T13:31:22.358848614Z\",\n      \"endTime\": \"2026-09-21T13:55:28.347264006Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svdhsdlg6rnfar064q74t4cdqc27f07 still exists (deletion dispatched at 2026-09-21T13:31:23Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001882150\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-bwv6gpm86sml\" to finish deleting: GET https://rp.j1617792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e63b539b-04f9-49eb-85fb-c1a75144fe9f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e63b539b-04f9-49eb-85fb-c1a75144fe9f\",\n      \"name\": \"e63b539b-04f9-49eb-85fb-c1a75144fe9f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T13:31:22.358848614Z\",\n      \"endTime\": \"2026-09-21T13:55:28.347264006Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svdhsdlg6rnfar064q74t4cdqc27f07 still exists (deletion dispatched at 2026-09-21T13:31:23Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"70dd516c1f43388fd837634cd97a1bc3e14e28dcbd18fcb7d7f0ac40d05fab74","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102005289971617792","occurred_at":"2026-09-21T12:00:55Z","failed_at":"e2e","signature_id":"7a988132cbbf7fdafe883067e106d606723d252d7544d78e7acef8d1f5e6086b","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0017ca3d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-noedge-qhw84s\" in resourcegroup=\"rg-np-noedge-qhw84s-v885pfcpwtk5\" to finish deleting: GET https://rp.j1617792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/efc6e6ce-36fa-452d-8d54-4c05769e580b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/efc6e6ce-36fa-452d-8d54-4c05769e580b\",\n      \"name\": \"efc6e6ce-36fa-452d-8d54-4c05769e580b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T13:27:18.580288973Z\",\n      \"endTime\": \"2026-09-21T13:51:18.327358984Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svdj78dqvttjrgmuj02247ille3g0fa still exists (deletion dispatched at 2026-09-21T13:27:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0017ca3d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-noedge-qhw84s\" in resourcegroup=\"rg-np-noedge-qhw84s-v885pfcpwtk5\" to finish deleting: GET https://rp.j1617792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/efc6e6ce-36fa-452d-8d54-4c05769e580b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/efc6e6ce-36fa-452d-8d54-4c05769e580b\",\n      \"name\": \"efc6e6ce-36fa-452d-8d54-4c05769e580b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T13:27:18.580288973Z\",\n      \"endTime\": \"2026-09-21T13:51:18.327358984Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svdj78dqvttjrgmuj02247ille3g0fa still exists (deletion dispatched at 2026-09-21T13:27:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"727addba98ebc16c9006f0ac787a54aa20faecc0f262b897287877117f5f06b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102005289971617792","occurred_at":"2026-09-21T12:00:55Z","failed_at":"alert","signature_id":"6b559ba77381ca05b488e6c7256ece0f37db96f55bdaf5ab305a334e7ea0750c","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T13:07:01Z\n  Ended:       2026-09-21T13:34:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1617792-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.251:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1617792/providers/microsoft.monitor/accounts/hcps-j1617792\", namespace=\"ocm-arohcpci01-2svdhspilu7hjev3eenqsfgat6bij060-t5x1q0q6n3v3g4d\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"8d9afbb2-38d4-42d1-9c00-d0aab022a452\"\n  Description: Pod ocm-arohcpci01-2svdhspilu7hjev3eenqsfgat6bij060-t5x1q0q6n3v3g4d/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-21T13:08:02Z\n  Ended:       2026-09-21T13:37:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1617792-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.251:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1617792/providers/microsoft.monitor/accounts/hcps-j1617792\", namespace=\"ocm-arohcpci01-2svdhspilu7hjev3eenqsfgat6bij060-t5x1q0q6n3v3g4d\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"8d9afbb2-38d4-42d1-9c00-d0aab022a452\"\n  Description: Pod ocm-arohcpci01-2svdhspilu7hjev3eenqsfgat6bij060-t5x1q0q6n3v3g4d/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"9ae4c1fd33409319cf2e741a83f0908d3876925217fbf494ec0ad00f45aa1881","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102005289971617792","occurred_at":"2026-09-21T12:00:55Z","failed_at":"e2e","signature_id":"b807ae0dd8954122969887a16b52eb7d0aba2605dde39a2017a233b4c46de2ec","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a9a210\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-pqkvx6\" in resourcegroup=\"rg-candidate-4-20-gbxxdszxc78c\" to finish deleting: GET https://rp.j1617792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c5c805fb-7493-492d-a6a5-b4880b792d21\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c5c805fb-7493-492d-a6a5-b4880b792d21\",\n      \"name\": \"c5c805fb-7493-492d-a6a5-b4880b792d21\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T13:10:40.626676784Z\",\n      \"endTime\": \"2026-09-21T13:34:42.971978588Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svdhq5fg59rsvkbro2s8ua6m21837cb still exists (deletion dispatched at 2026-09-21T13:10:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a9a210\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-pqkvx6\" in resourcegroup=\"rg-candidate-4-20-gbxxdszxc78c\" to finish deleting: GET https://rp.j1617792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c5c805fb-7493-492d-a6a5-b4880b792d21\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c5c805fb-7493-492d-a6a5-b4880b792d21\",\n      \"name\": \"c5c805fb-7493-492d-a6a5-b4880b792d21\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T13:10:40.626676784Z\",\n      \"endTime\": \"2026-09-21T13:34:42.971978588Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svdhq5fg59rsvkbro2s8ua6m21837cb still exists (deletion dispatched at 2026-09-21T13:10:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c8e5b034c676203f54a0c240fc1d32f14f718d30a5967b205c4058d5cb719448","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102005289971617792","occurred_at":"2026-09-21T12:00:55Z","failed_at":"alert","signature_id":"9eb20849ea3e6cbfdd789103f3230cc4456c7fc8f4c1d878d634fc5bb72337ae","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T13:19:24Z\n  Ended:       2026-09-21T13:25:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1617792-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.251:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1617792/providers/microsoft.monitor/accounts/services-j1617792\", namespace=\"klusterlet-2svdhspilu7hjev3eenqsfgat6bij060\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-jqkl2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2svdhspilu7hjev3eenqsfgat6bij060/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-21T13:19:24Z\n  Ended:       2026-09-21T13:25:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1617792-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.251:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1617792/providers/microsoft.monitor/accounts/services-j1617792\", namespace=\"klusterlet-2svdhspilu7hjev3eenqsfgat6bij060\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-jqkl2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2svdhspilu7hjev3eenqsfgat6bij060/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-21T13:19:24Z\n  Ended:       2026-09-21T13:25:25Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1617792-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.251:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1617792/providers/microsoft.monitor/accounts/services-j1617792\", namespace=\"klusterlet-2svdhspilu7hjev3eenqsfgat6bij060\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-jqkl2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2svdhspilu7hjev3eenqsfgat6bij060/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-21T13:19:24Z\n  Ended:       2026-09-21T13:25:25Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1617792-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.251:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1617792/providers/microsoft.monitor/accounts/services-j1617792\", namespace=\"klusterlet-2svdhspilu7hjev3eenqsfgat6bij060\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-jqkl2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2svdhspilu7hjev3eenqsfgat6bij060/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}},{"row_id":"dcf5a159ca31cdc0443de284804e480955c3488495ca3bce73840233fff8a768","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102005289971617792","occurred_at":"2026-09-21T12:00:55Z","failed_at":"alert","signature_id":"777c29daa977405aa9a86bc2fc1ac3546515b2b5d2a6da09c9cbc2008a5159c7","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T13:20:45Z\n  Ended:       2026-09-21T13:26:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1617792-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1617792-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"f5aec6ee193be3054569f8c16f8262436cf5ee1df45931ecfa6dc7e27e343640","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102005289971617792","occurred_at":"2026-09-21T12:00:55Z","failed_at":"e2e","signature_id":"dab3693d14d9782921b827fba6ee33d8f8f972733beb0092cacb20670d5fe48e","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009ec7e0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-b2z8n4v2rbk8\" to finish creating: GET https://rp.j1617792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/43a71945-dc9d-4839-beab-0c726c10fb56\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/43a71945-dc9d-4839-beab-0c726c10fb56\",\n      \"name\": \"43a71945-dc9d-4839-beab-0c726c10fb56\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T12:45:38.191499699Z\",\n      \"endTime\": \"2026-09-21T13:14:43.039772525Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009ec7e0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-b2z8n4v2rbk8\" to finish creating: GET https://rp.j1617792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/43a71945-dc9d-4839-beab-0c726c10fb56\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/43a71945-dc9d-4839-beab-0c726c10fb56\",\n      \"name\": \"43a71945-dc9d-4839-beab-0c726c10fb56\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T12:45:38.191499699Z\",\n      \"endTime\": \"2026-09-21T13:14:43.039772525Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fc7ca7b0cdd192ede9d723a91f748d4912a09afb41f6c704bed4fbe76aba9e38","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102005289971617792","occurred_at":"2026-09-21T12:00:55Z","failed_at":"alert","signature_id":"e6dc2ec44d3fc10cff6e42ee0d190fd24299cdce3098bdea21dc18fc2d960a6f","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T13:05:06Z\n  Ended:       2026-09-21T13:32:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j1617792-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.251:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1617792/providers/microsoft.monitor/accounts/hcps-j1617792\", namespace=\"ocm-arohcpci01-2svdhspilu7hjev3eenqsfgat6bij060-t5x1q0q6n3v3g4d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-jqkl2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2svdhspilu7hjev3eenqsfgat6bij060-t5x1q0q6n3v3g4d/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-21T13:16:04Z\n  Ended:       2026-09-21T13:33:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j1617792-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.251:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1617792/providers/microsoft.monitor/accounts/hcps-j1617792\", namespace=\"ocm-arohcpci01-2svdhspilu7hjev3eenqsfgat6bij060-t5x1q0q6n3v3g4d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-jqkl2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2svdhspilu7hjev3eenqsfgat6bij060-t5x1q0q6n3v3g4d/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102005251379826688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6995,"pr_state":"closed","pr_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","final_merged_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T12:00:52Z","started_at":"2026-09-21T12:00:52Z","completed_at":"2026-09-21T14:16:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T14:34:21.851034171Z","timing_metadata_checked_at":"2026-09-21T14:34:21.851034171Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T14:34:21.854428658Z","region_metadata_checked_at":"2026-09-21T14:34:21.854428658Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f18ba1c887e045e58245e60bdcde20d9d8467bdf0eb80d099e946c97b270c707","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102005251379826688","occurred_at":"2026-09-21T12:00:52Z","failed_at":"e2e","signature_id":"61ce1b1d7a72cb1f66c19b59f6b759051dbe08c72d27773b6738b472dd9edc9c","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011ea288\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-8pcqbbkmftxv\" to finish deleting: GET https://rp.j9826688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e747b3e9-5863-4e8f-ba53-4cc83e17d970\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e747b3e9-5863-4e8f-ba53-4cc83e17d970\",\n      \"name\": \"e747b3e9-5863-4e8f-ba53-4cc83e17d970\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T13:31:58.86771018Z\",\n      \"endTime\": \"2026-09-21T13:55:58.851482871Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svdicemqtje739suv9sg3jbh572m2o9 still exists (deletion dispatched at 2026-09-21T13:32:03Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011ea288\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-8pcqbbkmftxv\" to finish deleting: GET https://rp.j9826688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e747b3e9-5863-4e8f-ba53-4cc83e17d970\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e747b3e9-5863-4e8f-ba53-4cc83e17d970\",\n      \"name\": \"e747b3e9-5863-4e8f-ba53-4cc83e17d970\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T13:31:58.86771018Z\",\n      \"endTime\": \"2026-09-21T13:55:58.851482871Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svdicemqtje739suv9sg3jbh572m2o9 still exists (deletion dispatched at 2026-09-21T13:32:03Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102001525193707520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6965,"pr_state":"closed","pr_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","final_merged_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T11:46:31Z","started_at":"2026-09-21T11:46:31Z","completed_at":"2026-09-21T13:44:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T14:02:54.48348133Z","timing_metadata_checked_at":"2026-09-21T14:02:54.48348133Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T14:02:54.488076414Z","region_metadata_checked_at":"2026-09-21T14:02:54.488076414Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"0f2cd99b1879cd3a4a6ab7ee8c599c4486ab72d55fb6a2824f44ad8f40a5ebd1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102001525193707520","occurred_at":"2026-09-21T11:46:31Z","failed_at":"e2e","signature_id":"186ac175cc37e33f1eaa9e81000b9ad2fcf1670566cc6849f36ba3c782d224b2","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.000s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.000s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: ...\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}},{"row_id":"25adeb710d28cc6e8dbcaaf045b535895ac4be7ee309979df24fd3cf625ca5d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102001525193707520","occurred_at":"2026-09-21T11:46:31Z","failed_at":"alert","signature_id":"f63e68f4c2e5492a896313a116b1ee24fe442ffee137ffd63905390e8df42151","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T12:51:53Z\n  Ended:       2026-09-21T12:57:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3707520-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3707520-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"5399cf4a120153835219805c72f1218b969b3c892fddae6955eb97d39887da05","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102001525193707520","occurred_at":"2026-09-21T11:46:31Z","failed_at":"e2e","signature_id":"58aba0094d0afe8b75d434031889ce3b73351d056152994b3e8ac8306cc19fef","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001254198\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-5rgkd92gcd49\" to finish deleting: GET https://rp.j3707520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7143d5cd-5132-4f79-ba68-c7c14d6f7cca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7143d5cd-5132-4f79-ba68-c7c14d6f7cca\",\n      \"name\": \"7143d5cd-5132-4f79-ba68-c7c14d6f7cca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T13:03:15.195519989Z\",\n      \"endTime\": \"2026-09-21T13:27:24.675886986Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svd4hdg7vm35mc9053kr6ve1u4eet5p still exists (deletion dispatched at 2026-09-21T13:03:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001254198\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-5rgkd92gcd49\" to finish deleting: GET https://rp.j3707520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7143d5cd-5132-4f79-ba68-c7c14d6f7cca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7143d5cd-5132-4f79-ba68-c7c14d6f7cca\",\n      \"name\": \"7143d5cd-5132-4f79-ba68-c7c14d6f7cca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T13:03:15.195519989Z\",\n      \"endTime\": \"2026-09-21T13:27:24.675886986Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svd4hdg7vm35mc9053kr6ve1u4eet5p still exists (deletion dispatched at 2026-09-21T13:03:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"91bfd46624135b03beaf98597d8c9d89d22ea8c07fafb4b5cafb1957b6835303","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102001525193707520","occurred_at":"2026-09-21T11:46:31Z","failed_at":"e2e","signature_id":"52232f5a6a5fab9b6c2c1f6c93bf19fa6912a190989e4e1317cfe966fea5acd4","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001b820d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-s7mnx88qmfbp\" to finish deleting: GET https://rp.j3707520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/52fe4bd0-3d19-4a7b-afa3-9c2e011dc4ec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/52fe4bd0-3d19-4a7b-afa3-9c2e011dc4ec\",\n      \"name\": \"52fe4bd0-3d19-4a7b-afa3-9c2e011dc4ec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T12:41:21.139719293Z\",\n      \"endTime\": \"2026-09-21T13:05:29.745212922Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svd44ec66fqhn02vj2d96nl1c6qs77f still exists (deletion dispatched at 2026-09-21T12:41:26Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001b820d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-s7mnx88qmfbp\" to finish deleting: GET https://rp.j3707520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/52fe4bd0-3d19-4a7b-afa3-9c2e011dc4ec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/52fe4bd0-3d19-4a7b-afa3-9c2e011dc4ec\",\n      \"name\": \"52fe4bd0-3d19-4a7b-afa3-9c2e011dc4ec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T12:41:21.139719293Z\",\n      \"endTime\": \"2026-09-21T13:05:29.745212922Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svd44ec66fqhn02vj2d96nl1c6qs77f still exists (deletion dispatched at 2026-09-21T12:41:26Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"daf3af3a97551d25b495f8c626f4adb54ae04110b9df628310734039fe738f52","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102001525193707520","occurred_at":"2026-09-21T11:46:31Z","failed_at":"alert","signature_id":"1326c6fa7e2750d389265f8a3cb4327de2a7d9b092661cc44064cf1c020b4fbd","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-21T13:30:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j3707520-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-2rlm6\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-2rlm6 (namespace arobit) on cluster ci01-j3707520-svc has averaged 125.4% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102000884778012672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7036,"pr_state":"open","pr_sha":"5b4ed2bbcfd71a8855166afdecb608abdd6337cd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T11:43:58Z","started_at":"2026-09-21T11:43:58Z","completed_at":"2026-09-21T14:07:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T14:24:01.681751452Z","timing_metadata_checked_at":"2026-09-21T14:24:01.681751452Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T14:24:01.68533884Z","region_metadata_checked_at":"2026-09-21T14:24:01.68533884Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"1173471878633663c60dc81ed22d45a765cfc15d6e48047cd62af187f80b3572","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102000884778012672","occurred_at":"2026-09-21T11:43:58Z","failed_at":"e2e","signature_id":"5679c2af2cd624e4f7017d71ce8cdaa88cd991ef76ae38fc7a0e9cd080a8b37d","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001326180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-mtw7mm\" in resourcegroup=\"rg-np-skip-minor-mtw7mm-wsvmntwcrnk6\" to finish deleting: GET https://rp.j8012672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d47166b0-1e0e-40ac-971d-0769bb0dd3e2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d47166b0-1e0e-40ac-971d-0769bb0dd3e2\",\n      \"name\": \"d47166b0-1e0e-40ac-971d-0769bb0dd3e2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T13:24:43.758090309Z\",\n      \"endTime\": \"2026-09-21T13:48:49.428134555Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svdcv2vo5qf8d6j75ocvo8m99pv2ibk still exists (deletion dispatched at 2026-09-21T13:24:46Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001326180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-mtw7mm\" in resourcegroup=\"rg-np-skip-minor-mtw7mm-wsvmntwcrnk6\" to finish deleting: GET https://rp.j8012672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d47166b0-1e0e-40ac-971d-0769bb0dd3e2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d47166b0-1e0e-40ac-971d-0769bb0dd3e2\",\n      \"name\": \"d47166b0-1e0e-40ac-971d-0769bb0dd3e2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T13:24:43.758090309Z\",\n      \"endTime\": \"2026-09-21T13:48:49.428134555Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svdcv2vo5qf8d6j75ocvo8m99pv2ibk still exists (deletion dispatched at 2026-09-21T13:24:46Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"66cb5dd5156b04eb2c15065b45625163d0cef60aff402229f6546c40e33d6365","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102000884778012672","occurred_at":"2026-09-21T11:43:58Z","failed_at":"e2e","signature_id":"d181a2ec7077bf7c88b803ca896a8df982f80f8cc8649c136ed0dca944a23f3a","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00132a0c0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-z5lw9q\" in resourcegroup=\"rg-np-version-upgrade-z5lw9q-pq48qwbhvqng\" to finish deleting: GET https://rp.j8012672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/60b37214-abc5-402c-8009-25019e2b3800\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/60b37214-abc5-402c-8009-25019e2b3800\",\n      \"name\": \"60b37214-abc5-402c-8009-25019e2b3800\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T13:21:57.812948897Z\",\n      \"endTime\": \"2026-09-21T13:45:59.420888598Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svdd0gu6038tue65di04lkguja473rs still exists (deletion dispatched at 2026-09-21T13:21:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00132a0c0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-z5lw9q\" in resourcegroup=\"rg-np-version-upgrade-z5lw9q-pq48qwbhvqng\" to finish deleting: GET https://rp.j8012672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/60b37214-abc5-402c-8009-25019e2b3800\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/60b37214-abc5-402c-8009-25019e2b3800\",\n      \"name\": \"60b37214-abc5-402c-8009-25019e2b3800\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T13:21:57.812948897Z\",\n      \"endTime\": \"2026-09-21T13:45:59.420888598Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svdd0gu6038tue65di04lkguja473rs still exists (deletion dispatched at 2026-09-21T13:21:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9988aaa9b397efa1221fc1755b24808016ebc193ad63f472d084d365f4746ef6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102000884778012672","occurred_at":"2026-09-21T11:43:58Z","failed_at":"e2e","signature_id":"6efc6584b97a9fb60cc0617ccf95818f69a7fbee5f76c745a09d887aa5cab509","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00116c0d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-sxtl6w\" in resourcegroup=\"rg-zstream-upgrade-4-20-lf4jll9k9m82\" to finish deleting: GET https://rp.j8012672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b803dc9-51c6-4a3e-aca4-0a05b463e705\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b803dc9-51c6-4a3e-aca4-0a05b463e705\",\n      \"name\": \"6b803dc9-51c6-4a3e-aca4-0a05b463e705\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T13:08:12.400609819Z\",\n      \"endTime\": \"2026-09-21T13:32:18.776410847Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svdco6on49nvpigdfurfohpsb0kscna still exists (deletion dispatched at 2026-09-21T13:08:14Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00116c0d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-sxtl6w\" in resourcegroup=\"rg-zstream-upgrade-4-20-lf4jll9k9m82\" to finish deleting: GET https://rp.j8012672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b803dc9-51c6-4a3e-aca4-0a05b463e705\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b803dc9-51c6-4a3e-aca4-0a05b463e705\",\n      \"name\": \"6b803dc9-51c6-4a3e-aca4-0a05b463e705\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T13:08:12.400609819Z\",\n      \"endTime\": \"2026-09-21T13:32:18.776410847Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svdco6on49nvpigdfurfohpsb0kscna still exists (deletion dispatched at 2026-09-21T13:08:14Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e63064557ed21d45f33d0c024f507dd845d9a9863af2600c2005923437916fee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102000884778012672","occurred_at":"2026-09-21T11:43:58Z","failed_at":"e2e","signature_id":"71559231c43565422b7e7f33f89c1a819e937d24bd56a4f69f673e07db5257c0","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00190c180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-k49jcx2lp66n\" to finish deleting: GET https://rp.j8012672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48f759d4-4eb6-4f00-9e7c-e5da7e17efff\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48f759d4-4eb6-4f00-9e7c-e5da7e17efff\",\n      \"name\": \"48f759d4-4eb6-4f00-9e7c-e5da7e17efff\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T13:12:18.335875018Z\",\n      \"endTime\": \"2026-09-21T13:36:19.404523008Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svdcuomu57kbqpt4g0e583vms389pnk still exists (deletion dispatched at 2026-09-21T13:12:25Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00190c180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-k49jcx2lp66n\" to finish deleting: GET https://rp.j8012672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48f759d4-4eb6-4f00-9e7c-e5da7e17efff\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/48f759d4-4eb6-4f00-9e7c-e5da7e17efff\",\n      \"name\": \"48f759d4-4eb6-4f00-9e7c-e5da7e17efff\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T13:12:18.335875018Z\",\n      \"endTime\": \"2026-09-21T13:36:19.404523008Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svdcuomu57kbqpt4g0e583vms389pnk still exists (deletion dispatched at 2026-09-21T13:12:25Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7016/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101998376915570688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7016,"pr_state":"closed","pr_sha":"0cc06e8ea4b94c4b313f9293f6d25990b60f5b7b","final_merged_sha":"0cc06e8ea4b94c4b313f9293f6d25990b60f5b7b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T11:34:00Z","started_at":"2026-09-21T11:34:00Z","completed_at":"2026-09-21T13:54:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T14:13:32.048701968Z","timing_metadata_checked_at":"2026-09-21T14:13:32.048701968Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T14:13:32.051976757Z","region_metadata_checked_at":"2026-09-21T14:13:32.051976757Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"fa7c54e4117e49b65ec08970a30ffa130b8f3b6604e1edc57d13c36619379a83","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7016/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101998376915570688","occurred_at":"2026-09-21T11:34:00Z","failed_at":"e2e","signature_id":"f0710494d277dc1dd55cad6db7fd32f6e7f6fa61d73b9d043a14b76eeb177217","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014fe198\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-5p9hzd69stwf\" to finish deleting: GET https://rp.j5570688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a1e90f31-ccfc-4167-a029-7fcd2e50d427\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a1e90f31-ccfc-4167-a029-7fcd2e50d427\",\n      \"name\": \"a1e90f31-ccfc-4167-a029-7fcd2e50d427\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T13:13:51.354385545Z\",\n      \"endTime\": \"2026-09-21T13:38:00.28844207Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svd9duq0kg3iqnu0cmvl55vo06a2b1s still exists (deletion dispatched at 2026-09-21T13:13:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014fe198\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-5p9hzd69stwf\" to finish deleting: GET https://rp.j5570688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a1e90f31-ccfc-4167-a029-7fcd2e50d427\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a1e90f31-ccfc-4167-a029-7fcd2e50d427\",\n      \"name\": \"a1e90f31-ccfc-4167-a029-7fcd2e50d427\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T13:13:51.354385545Z\",\n      \"endTime\": \"2026-09-21T13:38:00.28844207Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svd9duq0kg3iqnu0cmvl55vo06a2b1s still exists (deletion dispatched at 2026-09-21T13:13:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101988054414659584","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6688,"pr_state":"open","pr_sha":"bc1782f9dc7294e7b0a04ac2096980eb77f2b8f8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T10:52:48Z","started_at":"2026-09-21T10:52:48Z","completed_at":"2026-09-21T11:13:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T11:28:41.022522801Z","timing_metadata_checked_at":"2026-09-21T11:28:41.022522801Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-21T11:28:41.025856789Z","region_metadata_checked_at":"2026-09-21T12:31:06.017100361Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"f6c3f87aa3c801d0ef5f20ed57a32edc9f09578a788762b0cefc293702743fa4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101988054414659584","occurred_at":"2026-09-21T10:52:48Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7031/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101987530910994432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7031,"pr_state":"open","pr_sha":"a6af8a6f4fa5f4a84a4cea1ddea63e410d5cd928","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T10:50:55Z","started_at":"2026-09-21T10:50:55Z","completed_at":"2026-09-21T11:02:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T11:07:55.09413627Z","timing_metadata_checked_at":"2026-09-21T11:07:55.09413627Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-21T11:07:55.096849058Z","region_metadata_checked_at":"2026-09-21T12:10:01.318871317Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"021a4ef88c5edcb744e830ecf43f0e17e5a48f7066259590181cb3c495f9e726","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7031/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101987530910994432","occurred_at":"2026-09-21T10:50:55Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101987460312469504","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6974,"pr_state":"closed","pr_sha":"fdb1b526d51e98ec4ae2b5a2fc9fae30dbe31d6d","final_merged_sha":"fdb1b526d51e98ec4ae2b5a2fc9fae30dbe31d6d","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T10:50:38Z","started_at":"2026-09-21T10:50:38Z","completed_at":"2026-09-21T11:14:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T11:28:41.500391982Z","timing_metadata_checked_at":"2026-09-21T11:28:41.500391982Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-21T11:28:41.503048272Z","region_metadata_checked_at":"2026-09-21T12:31:06.555281317Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"7b638a37da93caca9a1e34250fa1fb5699a3b53150f9beb420963e0739457a22","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101987460312469504","occurred_at":"2026-09-21T10:50:38Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101986636668604416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6965,"pr_state":"closed","pr_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","final_merged_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T10:47:21Z","started_at":"2026-09-21T10:47:21Z","completed_at":"2026-09-21T11:14:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T11:28:41.40908801Z","timing_metadata_checked_at":"2026-09-21T11:28:41.40908801Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-21T11:28:41.411783501Z","region_metadata_checked_at":"2026-09-21T12:31:06.400394719Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"0aaafc7585bb6f99754adce906455f2ab3b7996c78fe33f0d820f84d8fcb5509","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101986636668604416","occurred_at":"2026-09-21T10:47:21Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101975484274315264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6889,"pr_state":"open","pr_sha":"8dc48bd909ca622959c09b47e9d151f6fff23575","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T10:03:02Z","started_at":"2026-09-21T10:03:02Z","completed_at":"2026-09-21T11:01:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T11:07:54.96790124Z","timing_metadata_checked_at":"2026-09-21T11:07:54.96790124Z","region_metadata_state":"invalid","region_metadata_first_checked_at":"2026-09-21T11:07:54.971537824Z","region_metadata_checked_at":"2026-09-21T12:10:01.041454894Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"dec9d3d25a39f2806181ee57424a8bc3cc417e7a191d8f681f740df467d21cd2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101975484274315264","occurred_at":"2026-09-21T10:03:02Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7031/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101970018378977280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7031,"pr_state":"open","pr_sha":"a6af8a6f4fa5f4a84a4cea1ddea63e410d5cd928","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T09:41:19Z","started_at":"2026-09-21T09:41:19Z","completed_at":"2026-09-21T10:43:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T10:57:12.443714917Z","timing_metadata_checked_at":"2026-09-21T10:57:12.443714917Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T10:57:12.446182309Z","region_metadata_checked_at":"2026-09-21T10:57:12.446182309Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"13481ff473639813b51844ad73a245e60fd42561fbbac10cfe712a382549d45d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7031/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101970018378977280","occurred_at":"2026-09-21T09:41:19Z","failed_at":"provision","signature_id":"47638de8abbfa32d030a4bbf7c034ca306565b82c1a32135e2b3ffc38e1c9429","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-21T10:03:47.924Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-09-21T10:33:47.694Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-21T10:03:47.930202871Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8977280-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j8977280-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-09-21T10:03:49.993282994Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8977280-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j8977280-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-21T10:03:50.275505506Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8977280-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j8977280-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ec3c88183f7c5b398c8afd2006c96cb4a23574b2cddf879ce75375aa8f118f39","failure_pattern":"error running Shell Step, failed to execute shell command: {\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"\u003credacted-id\u003e\",\"resourceGroup\":\"hcp-underlay-ci01-j8977280-mgmt-1\",\"clusterName\":\"ci01-j897728","search_query":"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-21T10:03:47.930202871Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8977280-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j8977280-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-09-21T10:03:49.993282994Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8977280-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j8977280-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-21T10:03:50.275505506Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8977280-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j8977280-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ec3c88183f7c5b398c8afd2006c96cb4a23574b2cddf879ce75375aa8f118f39"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7003/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101968322814808064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7003,"pr_state":"open","pr_sha":"5ef6466e2a8ec2041b8b57cd5821ccdc91bbdd88","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T09:34:35Z","started_at":"2026-09-21T09:34:35Z","completed_at":"2026-09-21T10:47:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T10:57:12.425308677Z","timing_metadata_checked_at":"2026-09-21T10:57:12.425308677Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T10:57:12.428179168Z","region_metadata_checked_at":"2026-09-21T10:57:12.428179168Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"38ba55705d91cd4ea41ae622ca56535533eacebf8d43d11cb6c7d4fa1770d8b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7003/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101968322814808064","occurred_at":"2026-09-21T09:34:35Z","failed_at":"provision","signature_id":"91828d604d88c1907f4664cbc9f89c3c818ef7626c896966e9e84d0b34bf07f0","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-21T10:06:30.701Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-09-21T10:36:30.417Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-21T10:06:30.708972948Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4808064-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j4808064-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-09-21T10:06:33.118843713Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4808064-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j4808064-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-21T10:06:33.444184017Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4808064-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j4808064-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bc523a18cf6a83c7eedea62cad4828b40d64abab0c762578ce714f8b17df4a8e","failure_pattern":"error running Shell Step, failed to execute shell command: {\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"\u003credacted-id\u003e\",\"resourceGroup\":\"hcp-underlay-ci01-j4808064-mgmt-1\",\"clusterName\":\"ci01-j480806","search_query":"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-21T10:06:30.708972948Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4808064-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j4808064-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-09-21T10:06:33.118843713Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4808064-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j4808064-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-21T10:06:33.444184017Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4808064-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j4808064-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["bc523a18cf6a83c7eedea62cad4828b40d64abab0c762578ce714f8b17df4a8e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101965713999663104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7044,"pr_state":"closed","pr_sha":"0f625957a86b5fda4999ce65087e506bc2499c89","final_merged_sha":"0f625957a86b5fda4999ce65087e506bc2499c89","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T09:24:13Z","started_at":"2026-09-21T09:24:13Z","completed_at":"2026-09-21T11:29:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T11:38:42.325110562Z","timing_metadata_checked_at":"2026-09-21T11:38:42.325110562Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T11:38:42.328008052Z","region_metadata_checked_at":"2026-09-21T11:38:42.328008052Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"15d08db9f4178f78c868ce4978639e58cc5f070663fe9ba6f8cf18e9d6202625","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101965713999663104","occurred_at":"2026-09-21T09:24:13Z","failed_at":"e2e","signature_id":"95b8ff232a3a87f5dcc25e5121d0462b1d51ccd7abbaf5be0c856002753a39e9","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0018e0060\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-98cmmd\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-gvrwxl9vvpmg\" to finish deleting: GET https://rp.j9663104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4de7c54d-8700-4c66-ae32-c27713cd1d40\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4de7c54d-8700-4c66-ae32-c27713cd1d40\",\n      \"name\": \"4de7c54d-8700-4c66-ae32-c27713cd1d40\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:33:15.594049392Z\",\n      \"endTime\": \"2026-09-21T10:57:21.825655983Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svb4s0r2sehi5uvoqo9tjva522ujuc6 still exists (deletion dispatched at 2026-09-21T10:33:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0018e0060\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-98cmmd\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-gvrwxl9vvpmg\" to finish deleting: GET https://rp.j9663104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4de7c54d-8700-4c66-ae32-c27713cd1d40\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4de7c54d-8700-4c66-ae32-c27713cd1d40\",\n      \"name\": \"4de7c54d-8700-4c66-ae32-c27713cd1d40\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:33:15.594049392Z\",\n      \"endTime\": \"2026-09-21T10:57:21.825655983Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svb4s0r2sehi5uvoqo9tjva522ujuc6 still exists (deletion dispatched at 2026-09-21T10:33:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"16f3a5061f746c492414a7169489c3f90230eb0ad6504a6fe46876c1bc8cf8e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101965713999663104","occurred_at":"2026-09-21T09:24:13Z","failed_at":"e2e","signature_id":"8a03f8d7d8664fdc6926a2a751ab659535d2a1df5bacf20227df981409e4992b","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e3ab10\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-jcvfb4wmbljw\" to finish deleting: GET https://rp.j9663104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fdbca9d2-e9b2-491a-8405-4b0598064c63\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fdbca9d2-e9b2-491a-8405-4b0598064c63\",\n      \"name\": \"fdbca9d2-e9b2-491a-8405-4b0598064c63\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:31:21.582663948Z\",\n      \"endTime\": \"2026-09-21T10:55:21.85126634Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svb4o21n7e1d1vgdhv23sdiarqs1kr9 still exists (deletion dispatched at 2026-09-21T10:31:24Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e3ab10\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-jcvfb4wmbljw\" to finish deleting: GET https://rp.j9663104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fdbca9d2-e9b2-491a-8405-4b0598064c63\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fdbca9d2-e9b2-491a-8405-4b0598064c63\",\n      \"name\": \"fdbca9d2-e9b2-491a-8405-4b0598064c63\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:31:21.582663948Z\",\n      \"endTime\": \"2026-09-21T10:55:21.85126634Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svb4o21n7e1d1vgdhv23sdiarqs1kr9 still exists (deletion dispatched at 2026-09-21T10:31:24Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2c00eccc746b0c6c202966c1963d240a7534c944daa347fa6aaa79a9df9c431f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101965713999663104","occurred_at":"2026-09-21T09:24:13Z","failed_at":"e2e","signature_id":"344d2e1c3dd4d1231e5766f9ac88daeb1bcbe6f3d014923eff67ede021c7519f","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc002936270\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-t75p2qzhj8gw\" to finish deleting: GET https://rp.j9663104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f8ee760-75cf-435e-9fea-b1f68a59a646\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f8ee760-75cf-435e-9fea-b1f68a59a646\",\n      \"name\": \"7f8ee760-75cf-435e-9fea-b1f68a59a646\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:32:24.452769415Z\",\n      \"endTime\": \"2026-09-21T10:56:31.862771147Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svb5qjlj3rg0akksr0p4i913i1rl6a9 still exists (deletion dispatched at 2026-09-21T10:32:25Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc002936270\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-t75p2qzhj8gw\" to finish deleting: GET https://rp.j9663104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f8ee760-75cf-435e-9fea-b1f68a59a646\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f8ee760-75cf-435e-9fea-b1f68a59a646\",\n      \"name\": \"7f8ee760-75cf-435e-9fea-b1f68a59a646\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:32:24.452769415Z\",\n      \"endTime\": \"2026-09-21T10:56:31.862771147Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svb5qjlj3rg0akksr0p4i913i1rl6a9 still exists (deletion dispatched at 2026-09-21T10:32:25Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e64b90cec62f63329f93871c59d61e07cc8b5c08aa2675ee41c054dd146bab67","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101965713999663104","occurred_at":"2026-09-21T09:24:13Z","failed_at":"e2e","signature_id":"3228b928e838cd2a86309c39c89be0dd3e259b8e99d81bc8cf8621aac907a392","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011fe228\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-nnxpxcsf9rwh\" to finish deleting: GET https://rp.j9663104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d32bf532-2d4e-4d48-8778-8c230414a084\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d32bf532-2d4e-4d48-8778-8c230414a084\",\n      \"name\": \"d32bf532-2d4e-4d48-8778-8c230414a084\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:46:47.352616931Z\",\n      \"endTime\": \"2026-09-21T11:10:55.206232809Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svb4pu3gmd0hg24gfto1v4usp09dgoq still exists (deletion dispatched at 2026-09-21T10:46:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011fe228\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-nnxpxcsf9rwh\" to finish deleting: GET https://rp.j9663104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d32bf532-2d4e-4d48-8778-8c230414a084\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d32bf532-2d4e-4d48-8778-8c230414a084\",\n      \"name\": \"d32bf532-2d4e-4d48-8778-8c230414a084\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:46:47.352616931Z\",\n      \"endTime\": \"2026-09-21T11:10:55.206232809Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svb4pu3gmd0hg24gfto1v4usp09dgoq still exists (deletion dispatched at 2026-09-21T10:46:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e82ca99de7a2306a0ace32125f8bf849838bcfa8880aee81d6f7dff809c1e4be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101965713999663104","occurred_at":"2026-09-21T09:24:13Z","failed_at":"e2e","signature_id":"fab8da189775755ead4c6d20a595bf3352af47e75870090ceac831592729fc7c","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0017321c8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-c48zs6nq2v97\" to finish deleting: GET https://rp.j9663104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/87c826c1-5abf-4918-9848-d5f19a8f03cf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/87c826c1-5abf-4918-9848-d5f19a8f03cf\",\n      \"name\": \"87c826c1-5abf-4918-9848-d5f19a8f03cf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:30:58.008249279Z\",\n      \"endTime\": \"2026-09-21T10:55:01.859150512Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svb4uegc83e10fb4fvblrjm16f4oef1 still exists (deletion dispatched at 2026-09-21T10:31:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0017321c8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-c48zs6nq2v97\" to finish deleting: GET https://rp.j9663104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/87c826c1-5abf-4918-9848-d5f19a8f03cf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/87c826c1-5abf-4918-9848-d5f19a8f03cf\",\n      \"name\": \"87c826c1-5abf-4918-9848-d5f19a8f03cf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:30:58.008249279Z\",\n      \"endTime\": \"2026-09-21T10:55:01.859150512Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svb4uegc83e10fb4fvblrjm16f4oef1 still exists (deletion dispatched at 2026-09-21T10:31:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101965282229620736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7030,"pr_state":"closed","pr_sha":"50f9710d6eb949ad5cf7c325f7c4d6cce9083ddf","final_merged_sha":"50f9710d6eb949ad5cf7c325f7c4d6cce9083ddf","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T09:22:30Z","started_at":"2026-09-21T09:22:30Z","completed_at":"2026-09-21T11:24:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T11:38:41.892508058Z","timing_metadata_checked_at":"2026-09-21T11:38:41.892508058Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T11:38:41.895527549Z","region_metadata_checked_at":"2026-09-21T11:38:41.895527549Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"4521ddd46c4e739cf1f0ef3b174d2c43fd3fa4d87c6eefb8ef4f96c5b1a8a2ff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101965282229620736","occurred_at":"2026-09-21T09:22:30Z","failed_at":"alert","signature_id":"e07a4e80b0379a80d842e0bf9ec944110e439671b8351e6c41d23817d3c8e509","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T10:09:50Z\n  Ended:       2026-09-21T10:42:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j9620736-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j9620736-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101964585778024448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T09:19:44Z","started_at":"2026-09-21T09:19:44Z","completed_at":"2026-09-21T11:13:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T11:28:41.056685878Z","timing_metadata_checked_at":"2026-09-21T11:28:41.056685878Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T11:28:41.059312069Z","region_metadata_checked_at":"2026-09-21T11:28:41.059312069Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"18f6c061849cf1a4655741408966a57e1399064f4371dd8263941eab5d131c31","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101964585778024448","occurred_at":"2026-09-21T09:19:44Z","failed_at":"e2e","signature_id":"7e751e204f5bd1d1f0ec3bbc2c168cfad78021b60313990cb7a7a6cf058bd478","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001012228\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-vq2vgr\" in resourcegroup=\"rg-np-version-upgrade-vq2vgr-cckgpvz7pndd\" to finish deleting: GET https://rp.j8024448.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d481c4e3-dae9-468c-974a-c1944294aef6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d481c4e3-dae9-468c-974a-c1944294aef6\",\n      \"name\": \"d481c4e3-dae9-468c-974a-c1944294aef6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:22:43.251179723Z\",\n      \"endTime\": \"2026-09-21T10:46:46.294621181Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svau2tcj7pfh88pjia65i2kh1jul299 still exists (deletion dispatched at 2026-09-21T10:22:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001012228\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-vq2vgr\" in resourcegroup=\"rg-np-version-upgrade-vq2vgr-cckgpvz7pndd\" to finish deleting: GET https://rp.j8024448.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d481c4e3-dae9-468c-974a-c1944294aef6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d481c4e3-dae9-468c-974a-c1944294aef6\",\n      \"name\": \"d481c4e3-dae9-468c-974a-c1944294aef6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:22:43.251179723Z\",\n      \"endTime\": \"2026-09-21T10:46:46.294621181Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svau2tcj7pfh88pjia65i2kh1jul299 still exists (deletion dispatched at 2026-09-21T10:22:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7f1dc1a691506a1d0415399bcf5711fb9a501e2de65cc9b7a5a5aea8768e176c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101964585778024448","occurred_at":"2026-09-21T09:19:44Z","failed_at":"e2e","signature_id":"311965214f01e90fa04882c8fbe0ef17618e4a7143319fa2179f856e5e77b803","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001308498\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-noedge-8mrllx\" in resourcegroup=\"rg-np-noedge-8mrllx-82t2dsg9gzgw\" to finish deleting: GET https://rp.j8024448.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fee2636d-7191-4b7f-8841-eea73bc4141d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fee2636d-7191-4b7f-8841-eea73bc4141d\",\n      \"name\": \"fee2636d-7191-4b7f-8841-eea73bc4141d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:23:29.820724485Z\",\n      \"endTime\": \"2026-09-21T10:47:36.265381239Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svau2s1ko0hr4bf7ue7o4i7taa24scf still exists (deletion dispatched at 2026-09-21T10:23:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001308498\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-noedge-8mrllx\" in resourcegroup=\"rg-np-noedge-8mrllx-82t2dsg9gzgw\" to finish deleting: GET https://rp.j8024448.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fee2636d-7191-4b7f-8841-eea73bc4141d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fee2636d-7191-4b7f-8841-eea73bc4141d\",\n      \"name\": \"fee2636d-7191-4b7f-8841-eea73bc4141d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:23:29.820724485Z\",\n      \"endTime\": \"2026-09-21T10:47:36.265381239Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svau2s1ko0hr4bf7ue7o4i7taa24scf still exists (deletion dispatched at 2026-09-21T10:23:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d9077a762d9490b923a6b28262a2bb5b3de3bb8d5cb693e307a190c09841345d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101964585778024448","occurred_at":"2026-09-21T09:19:44Z","failed_at":"e2e","signature_id":"dd6a029c99d5ce5ac2800b1ebb00831cf2fb6526dda84a17d2d65e4aa5deeb10","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a6e090\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-tp9mrd4vg6gx\" to finish deleting: GET https://rp.j8024448.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4b888984-3115-4b39-bad6-fa2e3b75b3c3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4b888984-3115-4b39-bad6-fa2e3b75b3c3\",\n      \"name\": \"4b888984-3115-4b39-bad6-fa2e3b75b3c3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:31:26.068904427Z\",\n      \"endTime\": \"2026-09-21T10:55:27.606006007Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svau0q4s25jgnt3tgekea8cpbcbpeid still exists (deletion dispatched at 2026-09-21T10:31:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a6e090\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-tp9mrd4vg6gx\" to finish deleting: GET https://rp.j8024448.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4b888984-3115-4b39-bad6-fa2e3b75b3c3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4b888984-3115-4b39-bad6-fa2e3b75b3c3\",\n      \"name\": \"4b888984-3115-4b39-bad6-fa2e3b75b3c3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:31:26.068904427Z\",\n      \"endTime\": \"2026-09-21T10:55:27.606006007Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svau0q4s25jgnt3tgekea8cpbcbpeid still exists (deletion dispatched at 2026-09-21T10:31:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f5b578fb34c7aa3b613bc98a43d012a68e65d7c82afa8b8ad15c26f538c70922","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101964585778024448","occurred_at":"2026-09-21T09:19:44Z","failed_at":"alert","signature_id":"ae7cd9c22318f9a18fca10d6a8f425ab6f78913bb492f252d745398d16c30e1d","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-21T10:59:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j8024448-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-bft9m\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-bft9m (namespace arobit) on cluster ci01-j8024448-svc has averaged 126.8% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7034/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101963613219590144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7034,"pr_state":"closed","pr_sha":"125c1b478598f52a3fafb396836c61d736ef06aa","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T09:15:41Z","started_at":"2026-09-21T09:15:41Z","completed_at":"2026-09-21T11:26:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T11:38:42.045246365Z","timing_metadata_checked_at":"2026-09-21T11:38:42.045246365Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T11:38:42.048025456Z","region_metadata_checked_at":"2026-09-21T11:38:42.048025456Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"99361205f82eb1953332ccde4a4106e8a8a7dbf28b0e9c5f50692e300765aae4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7034/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101963613219590144","occurred_at":"2026-09-21T09:15:41Z","failed_at":"e2e","signature_id":"c66a2eef5d7c18708f4b72eff034e88c337ece780c087726a9d5780ed3b03adb","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00177e0a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-7bwgcmrlxdt4\" to finish deleting: GET https://rp.j9590144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6723973d-aa16-4f4b-a7a6-8b011a3abac4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6723973d-aa16-4f4b-a7a6-8b011a3abac4\",\n      \"name\": \"6723973d-aa16-4f4b-a7a6-8b011a3abac4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:45:43.224771957Z\",\n      \"endTime\": \"2026-09-21T11:09:44.667249123Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svb36j4tacoerfg1rjkpjh6sjsbs0rm still exists (deletion dispatched at 2026-09-21T10:45:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00177e0a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-7bwgcmrlxdt4\" to finish deleting: GET https://rp.j9590144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6723973d-aa16-4f4b-a7a6-8b011a3abac4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6723973d-aa16-4f4b-a7a6-8b011a3abac4\",\n      \"name\": \"6723973d-aa16-4f4b-a7a6-8b011a3abac4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:45:43.224771957Z\",\n      \"endTime\": \"2026-09-21T11:09:44.667249123Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svb36j4tacoerfg1rjkpjh6sjsbs0rm still exists (deletion dispatched at 2026-09-21T10:45:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101960313149591552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6995,"pr_state":"closed","pr_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","final_merged_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T09:02:45Z","started_at":"2026-09-21T09:02:45Z","completed_at":"2026-09-21T11:13:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T11:28:41.518616916Z","timing_metadata_checked_at":"2026-09-21T11:28:41.518616916Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T11:28:41.521463606Z","region_metadata_checked_at":"2026-09-21T11:28:41.521463606Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d3c7f9d396fe350455c3a61b30368db156d0bd7edfbb52e00e1ee495e765d49a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101960313149591552","occurred_at":"2026-09-21T09:02:45Z","failed_at":"e2e","signature_id":"af524ecd37c7482d375878b249fdb7c4534d2b38b5505bd4289fc03097149026","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b94ca0\u003e: \n    failed to create HCP cluster np-downgrade-78fkhm: failed waiting for cluster=\"np-downgrade-78fkhm\" in resourcegroup=\"rg-np-downgrade-78fkhm-mnlqzpfzdmbk\" to finish creating: GET https://rp.j9591552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4ed3e6b5-4161-4349-9abf-4a92c51749bd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4ed3e6b5-4161-4349-9abf-4a92c51749bd\",\n      \"name\": \"4ed3e6b5-4161-4349-9abf-4a92c51749bd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T09:54:06.067313752Z\",\n      \"endTime\": \"2026-09-21T10:23:08.770391006Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 7s ago; version 4.21.34 is Partial (want Completed), started 5m11s ago; version 4.21.34 is Partial (want Completed), started 10m32s ago; version 4.21.34 is Partial (want Completed), started 14m36s ago; version 4.21.34 is Partial (want Completed), started 19m54s ago; version 4.21.34 is Partial (want Completed), started 23m29s ago; version 4.21.34 is Partial (want Completed), started 24m30s ago; version 4.21.34 is Partial (want Completed), started 25m7s ago; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b94ca0\u003e: \n    failed to create HCP cluster np-downgrade-78fkhm: failed waiting for cluster=\"np-downgrade-78fkhm\" in resourcegroup=\"rg-np-downgrade-78fkhm-mnlqzpfzdmbk\" to finish creating: GET https://rp.j9591552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4ed3e6b5-4161-4349-9abf-4a92c51749bd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4ed3e6b5-4161-4349-9abf-4a92c51749bd\",\n      \"name\": \"4ed3e6b5-4161-4349-9abf-4a92c51749bd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T09:54:06.067313752Z\",\n      \"endTime\": \"2026-09-21T10:23:08.770391006Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 7s ago; version 4.21.34 is Partial (want Completed), started 5m11s ago; version 4.21.34 is Partial (want Completed), started 10m32s ago; version 4.21.34 is Partial (want Completed), started 14m36s ago; version 4.21.34 is Partial (want Completed), started 19m54s ago; version 4.21.34 is Partial (want Completed), started 23m29s ago; version 4.21.34 is Partial (want Completed), started 24m30s ago; version 4.21.34 is Partial (want Completed), started 25m7s ago; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"46425ffc9312d96c5106d1b27946c7775e4ddf6d0471e792a3aaf8c2bc4d3a86","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [openshift-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["46425ffc9312d96c5106d1b27946c7775e4ddf6d0471e792a3aaf8c2bc4d3a86"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101959670708047872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7033,"pr_state":"closed","pr_sha":"c448a0db20d77ea047c00534044b12cc2b2ac565","final_merged_sha":"ef821496d20065a3db001282a41d83827d994051","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T09:00:12Z","started_at":"2026-09-21T09:00:12Z","completed_at":"2026-09-21T09:43:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T09:52:55.491109898Z","timing_metadata_checked_at":"2026-09-21T09:52:55.491109898Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T09:52:55.494687285Z","region_metadata_checked_at":"2026-09-21T09:52:55.494687285Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"98a1a354db67e4cd09ab1ca70c12ed10f0ac2c8e400a0622e12c757d958f179f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101959670708047872","occurred_at":"2026-09-21T09:00:12Z","failed_at":"provision","signature_id":"09b1458b46da16205cd209ec402488a728d06d1e258b36845618ba3741f96692","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/infra-output","test_suite":"templatize-pipeline","failure_text":"time=2026-09-21T09:34:53.623Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output description=\"Step infra-output\\n  Kind: ARM\\n  Template: templates/svc-infra-lookup.bicep\\n  Parameters: configurations/svc-infra-lookup.tmpl.bicepparam\"\ntime=2026-09-21T09:34:54.045Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output\ntime=2026-09-21T09:34:54.556Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output deployment=4b9be9d2fe9447c77bb0d4b5c4352163b3563c2d648c6aabae0bd3896f58b810 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8047872-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F4b9be9d2fe9447c77bb0d4b5c4352163b3563c2d648c6aabae0bd3896f58b810\ntime=2026-09-21T09:35:24.831Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8047872-svc/providers/Microsoft.Resources/deployments/4b9be9d2fe9447c77bb0d4b5c4352163b3563c2d648c6aabae0bd3896f58b810/operationStatuses/08584116235912314952\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentActive\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/svc-kv-xehipxbjxawki' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '9/21/2026 9:34:50 AM' with correlationId 'f4ad37b9-8244-4bad-ac82-9eb55023bc2b', and will expire at '9/28/2026 9:34:50 AM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0008473d183c6e76249427529630789e6d4f9e8f1d5bdbea4b6bd57ae83c8f06","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentActive; detail message The deployment with resource id '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Resources/deployments/svc-kv-xehipxbjxawki' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '\u003ctimestamp\u003e' with correlationId '\u003cuuid\u003e', and will expire at '\u003ctimestamp\u003e' if it does not complete before then.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0008473d183c6e76249427529630789e6d4f9e8f1d5bdbea4b6bd57ae83c8f06"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7042/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101959649765888000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7042,"pr_state":"closed","pr_sha":"a576fbf17d1bfe631db730efd2a773060ec99279","final_merged_sha":"ed561d7657bd4aae970254bd806604f0b5a59888","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T09:00:07Z","started_at":"2026-09-21T09:00:07Z","completed_at":"2026-09-21T11:14:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T11:28:42.035891055Z","timing_metadata_checked_at":"2026-09-21T11:28:42.035891055Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T11:28:42.038366046Z","region_metadata_checked_at":"2026-09-21T11:28:42.038366046Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"2d79de6ab33d90d1718e3d8983e177c6c3220698075790ce70e27548a372326c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7042/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101959649765888000","occurred_at":"2026-09-21T09:00:07Z","failed_at":"e2e","signature_id":"649e2596090e91b495da9b9d5a33aa160be7802088e146e80777a9a6177b438c","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a80078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-fcbd7b\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-gm7xmqqlhk7q\" to finish deleting: GET https://rp.j5888000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d689087c-ffff-47e0-89eb-40b2969f5012\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d689087c-ffff-47e0-89eb-40b2969f5012\",\n      \"name\": \"d689087c-ffff-47e0-89eb-40b2969f5012\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:25:47.707680776Z\",\n      \"endTime\": \"2026-09-21T10:49:54.733565932Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svb382pcjghk7bnnr8flrmd78iq0hkt still exists (deletion dispatched at 2026-09-21T10:25:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a80078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-fcbd7b\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-gm7xmqqlhk7q\" to finish deleting: GET https://rp.j5888000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d689087c-ffff-47e0-89eb-40b2969f5012\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d689087c-ffff-47e0-89eb-40b2969f5012\",\n      \"name\": \"d689087c-ffff-47e0-89eb-40b2969f5012\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:25:47.707680776Z\",\n      \"endTime\": \"2026-09-21T10:49:54.733565932Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svb382pcjghk7bnnr8flrmd78iq0hkt still exists (deletion dispatched at 2026-09-21T10:25:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5da31f1694650da9108770283c75f31a9155ade2d7b3d78aa65dee244e43d4d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7042/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101959649765888000","occurred_at":"2026-09-21T09:00:07Z","failed_at":"e2e","signature_id":"c4b22eb13d5d92119b41c0b464177d3377e6621feb817f248b48ce47d9eb76bd","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0013502a0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-jf6q7n9qj5wj\" to finish deleting: GET https://rp.j5888000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ad5e9155-d0ec-4f8e-896c-95268fe5ae04\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ad5e9155-d0ec-4f8e-896c-95268fe5ae04\",\n      \"name\": \"ad5e9155-d0ec-4f8e-896c-95268fe5ae04\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:32:02.481353112Z\",\n      \"endTime\": \"2026-09-21T10:56:04.72514326Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svb32dn4up5rdijujhbjkta5l3fji8f still exists (deletion dispatched at 2026-09-21T10:32:03Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0013502a0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-jf6q7n9qj5wj\" to finish deleting: GET https://rp.j5888000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ad5e9155-d0ec-4f8e-896c-95268fe5ae04\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ad5e9155-d0ec-4f8e-896c-95268fe5ae04\",\n      \"name\": \"ad5e9155-d0ec-4f8e-896c-95268fe5ae04\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:32:02.481353112Z\",\n      \"endTime\": \"2026-09-21T10:56:04.72514326Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svb32dn4up5rdijujhbjkta5l3fji8f still exists (deletion dispatched at 2026-09-21T10:32:03Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"70dcd7b056e83b5ecbcdedc3f304cd9fc7ea51fc675e4206baeaacc206c302d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7042/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101959649765888000","occurred_at":"2026-09-21T09:00:07Z","failed_at":"e2e","signature_id":"f250f3ffcc754938cd306c7df6bc0b19fad59e75476a89f4a6f98c7161862d00","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00185e0f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-8mt5k2\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-5q6w5qxnjhh4\" to finish deleting: GET https://rp.j5888000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73803125-1a7f-4067-af20-3a6fa3e637e1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73803125-1a7f-4067-af20-3a6fa3e637e1\",\n      \"name\": \"73803125-1a7f-4067-af20-3a6fa3e637e1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:25:47.744819462Z\",\n      \"endTime\": \"2026-09-21T10:49:54.716137808Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svb2rbctdd0s6e1oshfr34443h3h21a still exists (deletion dispatched at 2026-09-21T10:25:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00185e0f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-8mt5k2\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-5q6w5qxnjhh4\" to finish deleting: GET https://rp.j5888000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73803125-1a7f-4067-af20-3a6fa3e637e1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73803125-1a7f-4067-af20-3a6fa3e637e1\",\n      \"name\": \"73803125-1a7f-4067-af20-3a6fa3e637e1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:25:47.744819462Z\",\n      \"endTime\": \"2026-09-21T10:49:54.716137808Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svb2rbctdd0s6e1oshfr34443h3h21a still exists (deletion dispatched at 2026-09-21T10:25:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c1ac9925dd815ce9221fb980a031ae3d3662682986f3b54c5cd3efe09e6c3b2f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7042/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101959649765888000","occurred_at":"2026-09-21T09:00:07Z","failed_at":"e2e","signature_id":"dde0e890b0e55309f0868d272c07009f87db193cf63f5360aa0ef3d3a561e003","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f8e060\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kwkrzd\" in resourcegroup=\"admin-credential-lifecycle-test-whczcrhsrl8z\" to finish deleting: GET https://rp.j5888000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f4c14adb-c4af-4f16-872c-81079fa0d7d0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f4c14adb-c4af-4f16-872c-81079fa0d7d0\",\n      \"name\": \"f4c14adb-c4af-4f16-872c-81079fa0d7d0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:23:04.873880064Z\",\n      \"endTime\": \"2026-09-21T10:47:04.760723886Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svb2pbsc21ete8ur0sfm3alfqiof3qc still exists (deletion dispatched at 2026-09-21T10:23:05Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f8e060\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kwkrzd\" in resourcegroup=\"admin-credential-lifecycle-test-whczcrhsrl8z\" to finish deleting: GET https://rp.j5888000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f4c14adb-c4af-4f16-872c-81079fa0d7d0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f4c14adb-c4af-4f16-872c-81079fa0d7d0\",\n      \"name\": \"f4c14adb-c4af-4f16-872c-81079fa0d7d0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:23:04.873880064Z\",\n      \"endTime\": \"2026-09-21T10:47:04.760723886Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svb2pbsc21ete8ur0sfm3alfqiof3qc still exists (deletion dispatched at 2026-09-21T10:23:05Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"daa6f2a8c0823a199b36ef44fd4de892f82290af5cfd9afd21e3944c52248039","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7042/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101959649765888000","occurred_at":"2026-09-21T09:00:07Z","failed_at":"e2e","signature_id":"4d21f17dd26a2b73258494d44acbfa78d8bd39a70ad88c5cec83184763e82288","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001096690\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-22-btwpkt\" in resourcegroup=\"rg-zstream-upgrade-4-22-89xddcclvkbf\" to finish deleting: GET https://rp.j5888000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc6c8e20-113e-4ba3-9544-b5d5f0e482b8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc6c8e20-113e-4ba3-9544-b5d5f0e482b8\",\n      \"name\": \"dc6c8e20-113e-4ba3-9544-b5d5f0e482b8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:27:51.838728083Z\",\n      \"endTime\": \"2026-09-21T10:51:54.711857296Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svb2pmt7c1h06jmec91u9vfiq0lg0o7 still exists (deletion dispatched at 2026-09-21T10:28:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001096690\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-22-btwpkt\" in resourcegroup=\"rg-zstream-upgrade-4-22-89xddcclvkbf\" to finish deleting: GET https://rp.j5888000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc6c8e20-113e-4ba3-9544-b5d5f0e482b8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc6c8e20-113e-4ba3-9544-b5d5f0e482b8\",\n      \"name\": \"dc6c8e20-113e-4ba3-9544-b5d5f0e482b8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:27:51.838728083Z\",\n      \"endTime\": \"2026-09-21T10:51:54.711857296Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svb2pmt7c1h06jmec91u9vfiq0lg0o7 still exists (deletion dispatched at 2026-09-21T10:28:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101959627087286272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7000,"pr_state":"closed","pr_sha":"e253ee7d7a59b02821f6339251ae15272721a63a","final_merged_sha":"e253ee7d7a59b02821f6339251ae15272721a63a","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T09:00:02Z","started_at":"2026-09-21T09:00:02Z","completed_at":"2026-09-21T11:17:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T11:28:41.92323716Z","timing_metadata_checked_at":"2026-09-21T11:28:41.92323716Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T11:28:41.925941051Z","region_metadata_checked_at":"2026-09-21T11:28:41.925941051Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e0c8894abbc8a3917760c5e36ab55d7999400ee733dd3eae37de4651480f9f26","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101959627087286272","occurred_at":"2026-09-21T09:00:02Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101957660935983104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7009,"pr_state":"open","pr_sha":"3ea999746a8bb5c1666c5bd5b4678a5c293d1c76","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T08:52:13Z","started_at":"2026-09-21T08:52:13Z","completed_at":"2026-09-21T11:07:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T11:18:17.802378158Z","timing_metadata_checked_at":"2026-09-21T11:18:17.802378158Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T11:18:17.805732047Z","region_metadata_checked_at":"2026-09-21T11:18:17.805732047Z"},"failed_at":["alert","e2e"],"failed_test_count":8,"occurrences":[{"row_id":"190ce5ef002610ab6aa3a1840ed85be2231fe888243050b140e083f5be76ec80","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101957660935983104","occurred_at":"2026-09-21T08:52:13Z","failed_at":"e2e","signature_id":"6a43a6862c0065cc253d14451ede5aa600fd28a06f0c5b76be4cfe2d972cd1f1","test_name":"KSM HCP Metrics kube_node_info metrics should be present in Azure Monitor for the happy-path cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:44]: failed to load service config\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00011d9e0\u003e: \n    failed to get region configuration for region \"westus3\": the cloud public is not found in the config\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:44]: failed to load service config\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00011d9e0\u003e: \n    failed to get region configuration for region \"westus3\": the cloud public is not found in the config\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f9e7f6695ae491515f7431c8ad469b8f7d13c78ca50db9a4a7889cb1520bbe61","failure_pattern":"failed to get region configuration for region \"westus3\": the cloud public is not found in the config","search_query":"failed to get region configuration for region \"westus3\": the cloud public is not found in the config"}},{"row_id":"3fd494e773f7bbe82d2e9b53e91961c7400c2cec5e98b115c69b2720f82ddb9d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101957660935983104","occurred_at":"2026-09-21T08:52:13Z","failed_at":"e2e","signature_id":"f0d5d8607de37cd7dec414f531b6feedfaf08f76d0dbcd2f8e851cb16c770aa3","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00123c660\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-22-cqs6pw\" in resourcegroup=\"rg-zstream-upgrade-4-22-bbtx7c7lpnwz\" to finish deleting: GET https://rp.j5983104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/68f2a0de-3b77-4ff9-8741-586548f5e49f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/68f2a0de-3b77-4ff9-8741-586548f5e49f\",\n      \"name\": \"68f2a0de-3b77-4ff9-8741-586548f5e49f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:03:13.722497121Z\",\n      \"endTime\": \"2026-09-21T10:27:17.753044758Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svap0kf2s60f4qlmcb9rk2bt16145nc still exists (deletion dispatched at 2026-09-21T10:03:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00123c660\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-22-cqs6pw\" in resourcegroup=\"rg-zstream-upgrade-4-22-bbtx7c7lpnwz\" to finish deleting: GET https://rp.j5983104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/68f2a0de-3b77-4ff9-8741-586548f5e49f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/68f2a0de-3b77-4ff9-8741-586548f5e49f\",\n      \"name\": \"68f2a0de-3b77-4ff9-8741-586548f5e49f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:03:13.722497121Z\",\n      \"endTime\": \"2026-09-21T10:27:17.753044758Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svap0kf2s60f4qlmcb9rk2bt16145nc still exists (deletion dispatched at 2026-09-21T10:03:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6e9ca1836c0d6b9deb0cc69b3f99f9348aede9d5689b3817d5255edad3cec41f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101957660935983104","occurred_at":"2026-09-21T08:52:13Z","failed_at":"e2e","signature_id":"2937ee7bb944b7ba440bbb1ec339d74fa5a7a0e472138a1166603654b3b2182c","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0015a8240\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-sfn5hpkk8vqt\" to finish deleting: GET https://rp.j5983104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c4dc90ca-9c30-42b8-86ef-406fd0b401fe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c4dc90ca-9c30-42b8-86ef-406fd0b401fe\",\n      \"name\": \"c4dc90ca-9c30-42b8-86ef-406fd0b401fe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:11:31.663814998Z\",\n      \"endTime\": \"2026-09-21T10:35:40.910284808Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svaol4gt1pjkhhbugdcpgmd3lpc2fsj still exists (deletion dispatched at 2026-09-21T10:11:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0015a8240\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-sfn5hpkk8vqt\" to finish deleting: GET https://rp.j5983104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c4dc90ca-9c30-42b8-86ef-406fd0b401fe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c4dc90ca-9c30-42b8-86ef-406fd0b401fe\",\n      \"name\": \"c4dc90ca-9c30-42b8-86ef-406fd0b401fe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:11:31.663814998Z\",\n      \"endTime\": \"2026-09-21T10:35:40.910284808Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svaol4gt1pjkhhbugdcpgmd3lpc2fsj still exists (deletion dispatched at 2026-09-21T10:11:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"77ad0142cd8072eddccb0a05c67b64d3ff3f86e6392122f1e40d97fe3cd8dd12","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101957660935983104","occurred_at":"2026-09-21T08:52:13Z","failed_at":"e2e","signature_id":"495dc8a6647cd99377afe81823924a2fbf20cc20c5625009d30e1fdd6b993d95","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0016d40c0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-jb4v95dwfjf9\" to finish deleting: GET https://rp.j5983104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0ba0b22c-8f26-429a-a5e6-2f803f417fb6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0ba0b22c-8f26-429a-a5e6-2f803f417fb6\",\n      \"name\": \"0ba0b22c-8f26-429a-a5e6-2f803f417fb6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:22:40.04734485Z\",\n      \"endTime\": \"2026-09-21T10:46:40.925610292Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svaovuh1mlfkervu4kcdhfjl31cbtv8 still exists (deletion dispatched at 2026-09-21T10:22:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0016d40c0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-jb4v95dwfjf9\" to finish deleting: GET https://rp.j5983104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0ba0b22c-8f26-429a-a5e6-2f803f417fb6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0ba0b22c-8f26-429a-a5e6-2f803f417fb6\",\n      \"name\": \"0ba0b22c-8f26-429a-a5e6-2f803f417fb6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:22:40.04734485Z\",\n      \"endTime\": \"2026-09-21T10:46:40.925610292Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svaovuh1mlfkervu4kcdhfjl31cbtv8 still exists (deletion dispatched at 2026-09-21T10:22:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bb19beb76c60578135c939d938c1f003890dade31c28dcabf2abfcf210371b98","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101957660935983104","occurred_at":"2026-09-21T08:52:13Z","failed_at":"e2e","signature_id":"e73e3e44e4bb274d31ed11942aa7c5ed0b09d80e5ec6eb71bb9812568b4c95ff","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.003s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc00039e080\u003e: \n    failed to load service config: failed to get region configuration for region \"westus3\": the cloud public is not found in the config\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.003s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc00039e080\u003e: \n    failed to load service config: failed to get region configuration for region \"westus3\": the cloud public is not found in the config\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5e63b93cb193f07a813e2f8208a703ccccf8dbeff5ce901215ed1595e41f3ee5","failure_pattern":"failed to load service config: failed to get region configuration for region \"westus3\": the cloud public is not found in the config","search_query":"failed to load service config: failed to get region configuration for region \"westus3\": the cloud public is not found in the config"}},{"row_id":"c18563c360f1f7fdb927dc0f9f946432ebe8a7e750e1aecc2d4664d66f953a58","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101957660935983104","occurred_at":"2026-09-21T08:52:13Z","failed_at":"alert","signature_id":"eda8ce4d8b7223c0554f4b92e2b2c59cf7ed8c9ef1c547dc0159f4b294d3f1e5","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-21T10:42:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j5983104-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-rxbk7\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-rxbk7 (namespace arobit) on cluster ci01-j5983104-svc has averaged 126.7% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"e9c94fde9807003944388441852b21a0ca086ca440b347d9a31d443dc5d4d4aa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101957660935983104","occurred_at":"2026-09-21T08:52:13Z","failed_at":"e2e","signature_id":"1ec0a63534ebb28267362c3dd6962bc5271dcaa1a515781cd2760e00aca45b68","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0008729a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-r75lvz\" in resourcegroup=\"rg-np-version-upgrade-r75lvz-vz7gd4t9qzt4\" to finish deleting: GET https://rp.j5983104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e5bf1662-1724-4aaf-bb66-e1c530aef6dc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e5bf1662-1724-4aaf-bb66-e1c530aef6dc\",\n      \"name\": \"e5bf1662-1724-4aaf-bb66-e1c530aef6dc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:15:34.40643872Z\",\n      \"endTime\": \"2026-09-21T10:39:40.910556229Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svaoivqknd5itbige5uvmrii4o1n5o4 still exists (deletion dispatched at 2026-09-21T10:15:35Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0008729a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-r75lvz\" in resourcegroup=\"rg-np-version-upgrade-r75lvz-vz7gd4t9qzt4\" to finish deleting: GET https://rp.j5983104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e5bf1662-1724-4aaf-bb66-e1c530aef6dc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e5bf1662-1724-4aaf-bb66-e1c530aef6dc\",\n      \"name\": \"e5bf1662-1724-4aaf-bb66-e1c530aef6dc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:15:34.40643872Z\",\n      \"endTime\": \"2026-09-21T10:39:40.910556229Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svaoivqknd5itbige5uvmrii4o1n5o4 still exists (deletion dispatched at 2026-09-21T10:15:35Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f1c1401ce9bcf96e56b4cda503a171d1745443e2ff449842dba19a1e023620a2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101957660935983104","occurred_at":"2026-09-21T08:52:13Z","failed_at":"e2e","signature_id":"9e92342ccd944d15d136c5e24dfec0e8421fa9e2e9130a93f934e681eab72ae8","test_name":"AMA Metrics service metrics from SVC and MGMT clusters should be present in Azure Monitor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/ama_metrics.go:68]: failed to load service config\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014eed00\u003e: \n    failed to get region configuration for region \"westus3\": the cloud public is not found in the config\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/ama_metrics.go:68]: failed to load service config\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014eed00\u003e: \n    failed to get region configuration for region \"westus3\": the cloud public is not found in the config\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f9e7f6695ae491515f7431c8ad469b8f7d13c78ca50db9a4a7889cb1520bbe61","failure_pattern":"failed to get region configuration for region \"westus3\": the cloud public is not found in the config","search_query":"failed to get region configuration for region \"westus3\": the cloud public is not found in the config"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["5e63b93cb193f07a813e2f8208a703ccccf8dbeff5ce901215ed1595e41f3ee5","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","f9e7f6695ae491515f7431c8ad469b8f7d13c78ca50db9a4a7889cb1520bbe61"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101954553699110912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5724,"pr_state":"closed","pr_sha":"016fdecacb5c2b53cdf4e1d7f9db9356d994bc7c","final_merged_sha":"02f55855ad2e600a742ce1b551002c7fbd872350","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T08:39:52Z","started_at":"2026-09-21T08:39:52Z","completed_at":"2026-09-21T10:43:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T10:57:12.2956448Z","timing_metadata_checked_at":"2026-09-21T10:57:12.2956448Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T10:57:12.299786887Z","region_metadata_checked_at":"2026-09-21T10:57:12.299786887Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"1b4265f5762dd74982baad757b9c1c5ad0fded275533773d5c54b4ae6d440395","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101954553699110912","occurred_at":"2026-09-21T08:39:52Z","failed_at":"e2e","signature_id":"0a7c114bf57a8653fdfa00931ae7b60d836bba5362bcc6c798eb27d036d8d13f","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c2c4c0\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-z7kkssjppzxn\" to finish creating: GET https://rp.j9110912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b42d1d15-2b65-453b-bd99-8dc808b1a981\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b42d1d15-2b65-453b-bd99-8dc808b1a981\",\n      \"name\": \"b42d1d15-2b65-453b-bd99-8dc808b1a981\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T09:25:09.210146348Z\",\n      \"endTime\": \"2026-09-21T09:54:10.812258416Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, catalog-operator, redhat-marketplace-catalog, hosted-cluster-config-operator, ingress-operator, olm-operator, redhat-operators-catalog, dns-operator, community-operators-catalog, csi-snapshot-controller-operator, certified-operators-catalog, cluster-version-operator, cluster-network-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c2c4c0\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-z7kkssjppzxn\" to finish creating: GET https://rp.j9110912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b42d1d15-2b65-453b-bd99-8dc808b1a981\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b42d1d15-2b65-453b-bd99-8dc808b1a981\",\n      \"name\": \"b42d1d15-2b65-453b-bd99-8dc808b1a981\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T09:25:09.210146348Z\",\n      \"endTime\": \"2026-09-21T09:54:10.812258416Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, catalog-operator, redhat-marketplace-catalog, hosted-cluster-config-operator, ingress-operator, olm-operator, redhat-operators-catalog, dns-operator, community-operators-catalog, csi-snapshot-controller-operator, certified-operators-catalog, cluster-version-operator, cluster-network-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"676e68f8a68616ac8c57dc0fe075f4a0d00c211dff572a92461d02984b57a771","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["676e68f8a68616ac8c57dc0fe075f4a0d00c211dff572a92461d02984b57a771"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101950652551794688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6886,"pr_state":"closed","pr_sha":"a6aa6a03a137ec55e3f4f5d06cc620e6bf6dd7f3","final_merged_sha":"a6aa6a03a137ec55e3f4f5d06cc620e6bf6dd7f3","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T08:24:21Z","started_at":"2026-09-21T08:24:21Z","completed_at":"2026-09-21T11:02:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T11:07:54.838662723Z","timing_metadata_checked_at":"2026-09-21T11:07:54.838662723Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T11:07:54.842582606Z","region_metadata_checked_at":"2026-09-21T11:07:54.842582606Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"0a913cdc2c108197068d04dbb69029d0bcfcff2befa272bc6b0aa0d769ec0f47","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101950652551794688","occurred_at":"2026-09-21T08:24:21Z","failed_at":"e2e","signature_id":"072f75db8fda213fe8e5547937cd896765fc564afeb34f2c7cb33ee664d82812","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.005s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0023f6980\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:51502-\u003e172.30.0.10:53: read: connection refused\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.005s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0023f6980\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:51502-\u003e172.30.0.10:53: read: connection refused\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02"}},{"row_id":"3c8b974118ecc0bc6e51e82482b3f9648beaa16982e0c8ace2c4cb2d6c3d872e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101950652551794688","occurred_at":"2026-09-21T08:24:21Z","failed_at":"e2e","signature_id":"0a74ab05e6434f2e0e0573ad6f7a896d924c8f03848120074f789cf2bec9a785","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000b741b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-kqtrqxcrhv82\" to finish deleting: GET https://rp.j1794688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1e43024-9a81-4304-b189-69cb0828f79b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1e43024-9a81-4304-b189-69cb0828f79b\",\n      \"name\": \"e1e43024-9a81-4304-b189-69cb0828f79b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:16:11.033673498Z\",\n      \"endTime\": \"2026-09-21T10:40:18.549152743Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svakhuv6julovvam753t2n5o7uehf3o still exists (deletion dispatched at 2026-09-21T10:16:12Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000b741b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-kqtrqxcrhv82\" to finish deleting: GET https://rp.j1794688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1e43024-9a81-4304-b189-69cb0828f79b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1e43024-9a81-4304-b189-69cb0828f79b\",\n      \"name\": \"e1e43024-9a81-4304-b189-69cb0828f79b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:16:11.033673498Z\",\n      \"endTime\": \"2026-09-21T10:40:18.549152743Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svakhuv6julovvam753t2n5o7uehf3o still exists (deletion dispatched at 2026-09-21T10:16:12Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6bf4dc3a7561083b71fcef461066eb05bbecece6439728217d9273754a0f66d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101950652551794688","occurred_at":"2026-09-21T08:24:21Z","failed_at":"e2e","signature_id":"e349a4a3bfc67d52e7fb08f8e23f91e34479ed684cc8f33f04ee5c75d7882ec7","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-m6qvz8\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009c12e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-t4d2m.apps.aro.b4j9u1f4z9l3l1o.5f4g.j1794688.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-t4d2m.apps.aro.b4j9u1f4z9l3l1o.5f4g.j1794688.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-t4d2m.apps.aro.b4j9u1f4z9l3l1o.5f4g.j1794688.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-m6qvz8\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009c12e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-t4d2m.apps.aro.b4j9u1f4z9l3l1o.5f4g.j1794688.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-t4d2m.apps.aro.b4j9u1f4z9l3l1o.5f4g.j1794688.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-t4d2m.apps.aro.b4j9u1f4z9l3l1o.5f4g.j1794688.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-28d65.apps.aro.o4p4d0n2f5l6i5t.cztj.j0999424.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-28d65.apps.aro.o4p4d0n2f5l6i5t.cztj.j0999424.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-28d65.apps.aro.o4p4d0n2f5l6i5t.cztj.j0999424.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"ce2b7d2b04a78f88e3f5597adac4276241cfa48f68a7d54e61c2fc771c8d4082","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101950652551794688","occurred_at":"2026-09-21T08:24:21Z","failed_at":"e2e","signature_id":"55e9ac0721dd283641a01c5317a416ae19196636a9b9916789c45463b619ef32","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001012198\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-fvwg79\" in resourcegroup=\"rg-np-version-upgrade-fvwg79-8tc9hq55lj5c\" to finish deleting: GET https://rp.j1794688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ae1fc2b5-879f-4350-9240-c43027591da4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ae1fc2b5-879f-4350-9240-c43027591da4\",\n      \"name\": \"ae1fc2b5-879f-4350-9240-c43027591da4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:13:24.500908877Z\",\n      \"endTime\": \"2026-09-21T10:37:28.531462691Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svakivopnrjbikmn670df9m0e5ebic7 still exists (deletion dispatched at 2026-09-21T10:13:25Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001012198\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-fvwg79\" in resourcegroup=\"rg-np-version-upgrade-fvwg79-8tc9hq55lj5c\" to finish deleting: GET https://rp.j1794688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ae1fc2b5-879f-4350-9240-c43027591da4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ae1fc2b5-879f-4350-9240-c43027591da4\",\n      \"name\": \"ae1fc2b5-879f-4350-9240-c43027591da4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T10:13:24.500908877Z\",\n      \"endTime\": \"2026-09-21T10:37:28.531462691Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svakivopnrjbikmn670df9m0e5ebic7 still exists (deletion dispatched at 2026-09-21T10:13:25Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101945646264094720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T08:04:28Z","started_at":"2026-09-21T08:04:28Z","completed_at":"2026-09-21T09:16:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T09:31:31.70816292Z","timing_metadata_checked_at":"2026-09-21T09:31:31.70816292Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T09:31:31.711468408Z","region_metadata_checked_at":"2026-09-21T09:31:31.711468408Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"4fc27d45f34f45fed2d0295c405ece020a7690771c4609346efd44ded78607c7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101945646264094720","occurred_at":"2026-09-21T08:04:28Z","failed_at":"provision","signature_id":"a0979f2448d1b88c3708ade1308fd4e7b8ce115a93226ed8df5db5900a1a5bfc","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-21T08:59:12.017Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/svc-infra.bicep\\n  Parameters: configurations/svc-infra.tmpl.bicepparam\"\ntime=2026-09-21T08:59:13.543Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra\ntime=2026-09-21T08:59:14.650Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra deployment=2380d2dc5c272e55d92eda779d891cdc0c622a661aa6f1f76f2e7d85c1855d68 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4094720-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F2380d2dc5c272e55d92eda779d891cdc0c622a661aa6f1f76f2e7d85c1855d68\ntime=2026-09-21T08:59:44.965Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4094720-svc/providers/Microsoft.Resources/deployments/2380d2dc5c272e55d92eda779d891cdc0c622a661aa6f1f76f2e7d85c1855d68/operationStatuses/08584116257315009198\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/svc-kv-6eqsxh5wya5uu\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ConflictError\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"A conflict occurred that prevented the operation from completing. The operation failed because the Key Vault '' changed from the point the operation began. This can happen if parallel operations are being performed on the Key Vault. To prevent this error, serialize the operations so that only one operation is performed on the Key Vault at a time. Follow this link for more information: https://go.microsoft.com/fwlink/?linkid=2147741\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3aa903deb973b4ed907f0f5c21dcaedfc5e057679f9f766029ba983e0911b934","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ConflictError; detail message A conflict occurred that prevented the operation from completing. The operation failed because the Key Vault '' changed from the point the operation began. This can happen if parallel operations are being performed on the Key Vault. To prevent this error, serialize the operations so that only one operation is performed on the Key Vault at a time. Follow this link for more information: \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["3aa903deb973b4ed907f0f5c21dcaedfc5e057679f9f766029ba983e0911b934"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101942310727061504","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T07:51:13Z","started_at":"2026-09-21T07:51:13Z","completed_at":"2026-09-21T08:00:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T08:07:13.022160465Z","timing_metadata_checked_at":"2026-09-21T08:07:13.022160465Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-21T08:07:13.025463355Z","region_metadata_checked_at":"2026-09-21T09:10:59.17503242Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"68f1f060a0c2ca10abc2b9cb4d9b48da995ccb5a857f2abfefbf47198294b977","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101942310727061504","occurred_at":"2026-09-21T07:51:13Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101940890745442304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7033,"pr_state":"closed","pr_sha":"c448a0db20d77ea047c00534044b12cc2b2ac565","final_merged_sha":"ef821496d20065a3db001282a41d83827d994051","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T07:45:35Z","started_at":"2026-09-21T07:45:35Z","completed_at":"2026-09-21T07:54:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T08:07:13.167533724Z","timing_metadata_checked_at":"2026-09-21T08:07:13.167533724Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-21T08:07:13.170145116Z","region_metadata_checked_at":"2026-09-21T09:10:59.266589622Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"6720b42c9da60c2b38416f719b41ecdf87775b14572bf05fd8eff6168677fe78","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101940890745442304","occurred_at":"2026-09-21T07:45:35Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7042/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101925644194549760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7042,"pr_state":"closed","pr_sha":"a576fbf17d1bfe631db730efd2a773060ec99279","final_merged_sha":"ed561d7657bd4aae970254bd806604f0b5a59888","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T06:45:00Z","started_at":"2026-09-21T06:45:00Z","completed_at":"2026-09-21T08:45:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T09:00:15.980642304Z","timing_metadata_checked_at":"2026-09-21T09:00:15.980642304Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T09:00:15.983814894Z","region_metadata_checked_at":"2026-09-21T09:00:15.983814894Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"15f86919e19998f746467e6d61e260b3fd7fd1e88f17355085baba963035a42e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7042/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101925644194549760","occurred_at":"2026-09-21T06:45:00Z","failed_at":"e2e","signature_id":"97fa2646f2b54c74fd86fca3fe700cc51b19d211e3f27bc15f6e594664d36a4c","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000914288\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-r44lgj\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-vtns4zqt5kzm\" to finish deleting: GET https://rp.j4549760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2472d0ca-ba58-46fd-acd2-8ed3a089c1ce\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2472d0ca-ba58-46fd-acd2-8ed3a089c1ce\",\n      \"name\": \"2472d0ca-ba58-46fd-acd2-8ed3a089c1ce\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T08:01:06.657072109Z\",\n      \"endTime\": \"2026-09-21T08:25:11.418167296Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv8uap75mb29304n1l7bh2nvgaqpb27 still exists (deletion dispatched at 2026-09-21T08:01:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000914288\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-r44lgj\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-vtns4zqt5kzm\" to finish deleting: GET https://rp.j4549760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2472d0ca-ba58-46fd-acd2-8ed3a089c1ce\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2472d0ca-ba58-46fd-acd2-8ed3a089c1ce\",\n      \"name\": \"2472d0ca-ba58-46fd-acd2-8ed3a089c1ce\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T08:01:06.657072109Z\",\n      \"endTime\": \"2026-09-21T08:25:11.418167296Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv8uap75mb29304n1l7bh2nvgaqpb27 still exists (deletion dispatched at 2026-09-21T08:01:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2309b9d5d9bc7a95ba0426ad70736735057f1538dc662888e1249e86b4df0eef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7042/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101925644194549760","occurred_at":"2026-09-21T06:45:00Z","failed_at":"alert","signature_id":"6fc3aa3ba9dae5c77e867a5df73aa1503b7195ea3d32e0dc856c9d520fd81e87","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T08:01:39Z\n  Ended:       2026-09-21T08:07:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4549760-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4549760-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"cc4f3ea023a9083cfdfbf049e5a9e50e237b7ef84c1a57f0f56f7942fe0392ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7042/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101925644194549760","occurred_at":"2026-09-21T06:45:00Z","failed_at":"alert","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T07:46:48Z\n  Ended:       2026-09-21T07:52:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4549760-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"fb1873f903ea9f73d786a055adb0cf146039c3ab0fe1636491de73ef56e3122b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7042/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101925644194549760","occurred_at":"2026-09-21T06:45:00Z","failed_at":"e2e","signature_id":"cba95bd6892d6440e977ddbd13905a5d526c1ab2d57b88b6bbfc2576274972d2","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000cd23c0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-f4f4sxhwtxng\" to finish deleting: GET https://rp.j4549760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a41981a0-b1ef-4211-a480-6874ed7415c2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a41981a0-b1ef-4211-a480-6874ed7415c2\",\n      \"name\": \"a41981a0-b1ef-4211-a480-6874ed7415c2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T08:00:51.346750685Z\",\n      \"endTime\": \"2026-09-21T08:24:51.406875941Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv8uaoia279h9rgckjotdu8iu3pqjob still exists (deletion dispatched at 2026-09-21T08:00:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000cd23c0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-f4f4sxhwtxng\" to finish deleting: GET https://rp.j4549760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a41981a0-b1ef-4211-a480-6874ed7415c2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a41981a0-b1ef-4211-a480-6874ed7415c2\",\n      \"name\": \"a41981a0-b1ef-4211-a480-6874ed7415c2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T08:00:51.346750685Z\",\n      \"endTime\": \"2026-09-21T08:24:51.406875941Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv8uaoia279h9rgckjotdu8iu3pqjob still exists (deletion dispatched at 2026-09-21T08:00:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101914215290769408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5724,"pr_state":"closed","pr_sha":"016fdecacb5c2b53cdf4e1d7f9db9356d994bc7c","final_merged_sha":"02f55855ad2e600a742ce1b551002c7fbd872350","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T05:59:35Z","started_at":"2026-09-21T05:59:35Z","completed_at":"2026-09-21T08:07:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T08:18:08.819189635Z","timing_metadata_checked_at":"2026-09-21T08:18:08.819189635Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T08:18:08.822559621Z","region_metadata_checked_at":"2026-09-21T08:18:08.822559621Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"65b59377191e8e6cda34c6ab85f94df92d0474fe52db33f264b97547d60c2972","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101914215290769408","occurred_at":"2026-09-21T05:59:35Z","failed_at":"e2e","signature_id":"7c0fc3ed316118f6e5f618d7c54fdaec203e1cf74c0aeefa2148d2e8ae99edc7","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000b2c2a0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-bt4gcbb44v4k\" to finish deleting: GET https://rp.j0769408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/25514327-56c7-4568-b715-8086fd214fd3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/25514327-56c7-4568-b715-8086fd214fd3\",\n      \"name\": \"25514327-56c7-4568-b715-8086fd214fd3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T07:26:44.394306512Z\",\n      \"endTime\": \"2026-09-21T07:50:49.62279615Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv86rnc3vjq93289g9c7tuo24gitvt7 still exists (deletion dispatched at 2026-09-21T07:26:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000b2c2a0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-bt4gcbb44v4k\" to finish deleting: GET https://rp.j0769408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/25514327-56c7-4568-b715-8086fd214fd3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/25514327-56c7-4568-b715-8086fd214fd3\",\n      \"name\": \"25514327-56c7-4568-b715-8086fd214fd3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T07:26:44.394306512Z\",\n      \"endTime\": \"2026-09-21T07:50:49.62279615Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv86rnc3vjq93289g9c7tuo24gitvt7 still exists (deletion dispatched at 2026-09-21T07:26:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101911584593940480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T05:49:08Z","started_at":"2026-09-21T05:49:08Z","completed_at":"2026-09-21T07:42:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T07:56:43.320107008Z","timing_metadata_checked_at":"2026-09-21T07:56:43.320107008Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T07:56:43.323403998Z","region_metadata_checked_at":"2026-09-21T07:56:43.323403998Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"40d36995d49ac99e9a76fcce50d03dfcf9141c78d47883e477284911f42857ab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101911584593940480","occurred_at":"2026-09-21T05:49:08Z","failed_at":"e2e","signature_id":"a38c38772c571c44e8f83252d8c85e1ca7e0eeda32f392cc4e29fdecb6d64903","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001364090\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-xr4f8xjtqqz7\" to finish deleting: GET https://rp.j3940480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/68c23833-c73a-48e8-aed4-9afe871f1021\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/68c23833-c73a-48e8-aed4-9afe871f1021\",\n      \"name\": \"68c23833-c73a-48e8-aed4-9afe871f1021\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T07:02:55.756717366Z\",\n      \"endTime\": \"2026-09-21T07:27:02.075354063Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv7sub4ugq6anc6dbktsdtv4jknqduj still exists (deletion dispatched at 2026-09-21T07:02:57Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001364090\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-xr4f8xjtqqz7\" to finish deleting: GET https://rp.j3940480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/68c23833-c73a-48e8-aed4-9afe871f1021\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/68c23833-c73a-48e8-aed4-9afe871f1021\",\n      \"name\": \"68c23833-c73a-48e8-aed4-9afe871f1021\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T07:02:55.756717366Z\",\n      \"endTime\": \"2026-09-21T07:27:02.075354063Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv7sub4ugq6anc6dbktsdtv4jknqduj still exists (deletion dispatched at 2026-09-21T07:02:57Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4b053d1fef46666708c350eaaad64c15006bbd117a5eca18f8bba5023bd2f844","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101911584593940480","occurred_at":"2026-09-21T05:49:08Z","failed_at":"alert","signature_id":"72a8b9a5b51e1fb9c95f9604613babb551a08107cd32c60030d07687ad107163","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T06:21:35Z\n  Ended:       2026-09-21T06:55:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3940480-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3940480-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101901188189392896","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6439,"pr_state":"closed","pr_sha":"b6b00cdc8ea3ee249a9faa811955e4eb095652ef","final_merged_sha":"8e51fd56182fb53110559c6686373da1a4435f34","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T05:07:49Z","started_at":"2026-09-21T05:07:49Z","completed_at":"2026-09-21T07:22:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T07:35:41.004263682Z","timing_metadata_checked_at":"2026-09-21T07:35:41.004263682Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T07:35:41.008420368Z","region_metadata_checked_at":"2026-09-21T07:35:41.008420368Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"74296dec49c8b780428f071252825e1ea2ba27498f6b8254674527fcbb172397","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101901188189392896","occurred_at":"2026-09-21T05:07:49Z","failed_at":"e2e","signature_id":"221d7d581523fb43f667bb794fd7bb2762637883f1ecd66b815d35a3798cf05c","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0015460d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-fsb8r7fzfgj9\" to finish deleting: GET https://rp.j9392896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4907893e-292c-49a0-be95-954694e0081a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4907893e-292c-49a0-be95-954694e0081a\",\n      \"name\": \"4907893e-292c-49a0-be95-954694e0081a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T06:39:12.772961096Z\",\n      \"endTime\": \"2026-09-21T07:03:15.983730935Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv7gejc46t1vcjds4esh5obpresl6nh still exists (deletion dispatched at 2026-09-21T06:39:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0015460d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-fsb8r7fzfgj9\" to finish deleting: GET https://rp.j9392896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4907893e-292c-49a0-be95-954694e0081a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4907893e-292c-49a0-be95-954694e0081a\",\n      \"name\": \"4907893e-292c-49a0-be95-954694e0081a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T06:39:12.772961096Z\",\n      \"endTime\": \"2026-09-21T07:03:15.983730935Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv7gejc46t1vcjds4esh5obpresl6nh still exists (deletion dispatched at 2026-09-21T06:39:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9f9b3ac0e0bce76a8dc83920357eaaa1064bd2f11197ba5dbc3f2bf7f325dd48","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101901188189392896","occurred_at":"2026-09-21T05:07:49Z","failed_at":"alert","signature_id":"c56e16510643a5528d9ca10076ccb7914d0c60c0e2108546c9d1301313c563a5","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T06:15:25Z\n  Ended:       2026-09-21T06:49:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j9392896-svc\", component=\"frontend\", method=\"get\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n  Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} exceeded 1 second over the past 30 minutes (cluster ci01-j9392896-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101894657490292736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T04:41:52Z","started_at":"2026-09-21T04:41:52Z","completed_at":"2026-09-21T05:23:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T05:40:26.907053954Z","timing_metadata_checked_at":"2026-09-21T05:40:26.907053954Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T05:40:26.909892845Z","region_metadata_checked_at":"2026-09-21T05:40:26.909892845Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"00136c440b55d71248bccaea9f131c148c831a913ebe851b2aaf8ff993f53260","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101894657490292736","occurred_at":"2026-09-21T04:41:52Z","failed_at":"provision","signature_id":"175313ad0fc9f5748bbd51b4caad9e93581c246d348fafeb931df7a611c56303","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/admin-api-cert","test_suite":"templatize-pipeline","failure_text":"time=2026-09-21T05:07:29.651Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert description=\"Step admin-api-cert\\n Kind: CreateCertificate\\n\"\ntime=2026-09-21T05:07:29.651Z level=INFO msg=\"Creating certificate\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert vaultBaseUrl=https://aro-hcp-dev-svc-kv.vault.azure.net/ certificateName=admin-api-cert-ci01-j0292736 issuer=Self commonName=admin.j0292736.hcpsvc.osadev.cloud san=admin.j0292736.hcpsvc.osadev.cloud\ntime=2026-09-21T05:07:30.521Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert err=\"error running Create Certificate Step: failed to create certificate \\\"admin-api-cert-ci01-j0292736\\\" in vault \\\"https://aro-hcp-dev-svc-kv.vault.azure.net/\\\": POST https://aro-hcp-dev-svc-kv.vault.azure.net/certificates/admin-api-cert-ci01-j0292736/create\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Certificate admin-api-cert-ci01-j0292736 is currently in a deleted but recoverable state, and its name cannot be reused; in this state, the certificate can only be recovered or purged.\\\",\\n    \\\"innererror\\\": {\\n      \\\"code\\\": \\\"ObjectIsDeletedButRecoverable\\\"\\n    }\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6f6e7e66ad8bc42bd6647f1e246b1e77205441e0c9495974d9b1f940d8bdec9c","failure_pattern":"ERROR CODE: Conflict; detail code ObjectIsDeletedButRecoverable","search_query":"ERROR CODE: Conflict"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6f6e7e66ad8bc42bd6647f1e246b1e77205441e0c9495974d9b1f940d8bdec9c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101892420403728384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7032,"pr_state":"closed","pr_sha":"c17bf1b8be7e620f98afeeb00dde43c86e8ebf20","final_merged_sha":"c17bf1b8be7e620f98afeeb00dde43c86e8ebf20","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T04:29:27Z","started_at":"2026-09-21T04:29:27Z","completed_at":"2026-09-21T06:45:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T06:53:43.419244329Z","timing_metadata_checked_at":"2026-09-21T06:53:43.419244329Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T06:53:43.422591316Z","region_metadata_checked_at":"2026-09-21T06:53:43.422591316Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"58be5a68ba983696995543474f146486e76e1bb271e620c207c7efa97b1c3a74","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101892420403728384","occurred_at":"2026-09-21T04:29:27Z","failed_at":"e2e","signature_id":"158fed3e1f3ab673a73aeee045563f7eb6ad7f72564d212c6cbee3fe07d887fd","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001830de0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-tmn6sjfksxv8\" to finish deleting: GET https://rp.j3728384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b4b07814-f95c-4538-9aa4-ab89b2035089\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b4b07814-f95c-4538-9aa4-ab89b2035089\",\n      \"name\": \"b4b07814-f95c-4538-9aa4-ab89b2035089\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T06:00:53.538768832Z\",\n      \"endTime\": \"2026-09-21T06:24:56.291656829Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv73i1t9ri5hjnomo09g7sg4q952iua still exists (deletion dispatched at 2026-09-21T06:00:55Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001830de0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-tmn6sjfksxv8\" to finish deleting: GET https://rp.j3728384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b4b07814-f95c-4538-9aa4-ab89b2035089\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b4b07814-f95c-4538-9aa4-ab89b2035089\",\n      \"name\": \"b4b07814-f95c-4538-9aa4-ab89b2035089\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T06:00:53.538768832Z\",\n      \"endTime\": \"2026-09-21T06:24:56.291656829Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv73i1t9ri5hjnomo09g7sg4q952iua still exists (deletion dispatched at 2026-09-21T06:00:55Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101884774271422464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7000,"pr_state":"closed","pr_sha":"e253ee7d7a59b02821f6339251ae15272721a63a","final_merged_sha":"e253ee7d7a59b02821f6339251ae15272721a63a","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T04:01:57Z","started_at":"2026-09-21T04:01:57Z","completed_at":"2026-09-21T06:00:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T06:11:55.989363848Z","timing_metadata_checked_at":"2026-09-21T06:11:55.989363848Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T06:11:55.992251138Z","region_metadata_checked_at":"2026-09-21T06:11:55.992251138Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"8f8681920b8e7c1fed5f73ed0f706c2133085eb23c5cc14dd5fc396f41b6e6ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101884774271422464","occurred_at":"2026-09-21T04:01:57Z","failed_at":"alert","signature_id":"e66f8bba76d2fce89aae8974bc46ee9c31b4628037c24a399492092fbd22358d","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-21T05:47:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j1422464-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-v8ss6\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-v8ss6 (namespace arobit) on cluster ci01-j1422464-svc has averaged 125.4% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6092/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101882720345919488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6092,"pr_state":"open","pr_sha":"0b8aa60aa06ccf061b10ca69badf102c0520d2fe","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T03:54:26Z","started_at":"2026-09-21T03:54:26Z","completed_at":"2026-09-21T06:02:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T06:11:55.835739765Z","timing_metadata_checked_at":"2026-09-21T06:11:55.835739765Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T06:11:55.839637452Z","region_metadata_checked_at":"2026-09-21T06:11:55.839637452Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"626e568a4af0fc7fc90647fd161a41baaaca404a5b2ae94aaf897c90b7d4a608","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6092/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101882720345919488","occurred_at":"2026-09-21T03:54:26Z","failed_at":"e2e","signature_id":"9e1472e2b06c814ed8a5d578ca97b4a96a230638d41fe82e1e71290620796625","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0009800a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-rczjfj\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-6m2k6f6cxbxw\" to finish deleting: GET https://rp.j5919488.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e5de7dae-f611-48d5-9887-bd471d4a782f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e5de7dae-f611-48d5-9887-bd471d4a782f\",\n      \"name\": \"e5de7dae-f611-48d5-9887-bd471d4a782f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T05:03:30.893249828Z\",\n      \"endTime\": \"2026-09-21T05:27:33.685762872Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv6ctdi2qdv7iqjg03ojai036om0fl6 still exists (deletion dispatched at 2026-09-21T05:03:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0009800a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-rczjfj\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-6m2k6f6cxbxw\" to finish deleting: GET https://rp.j5919488.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e5de7dae-f611-48d5-9887-bd471d4a782f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e5de7dae-f611-48d5-9887-bd471d4a782f\",\n      \"name\": \"e5de7dae-f611-48d5-9887-bd471d4a782f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T05:03:30.893249828Z\",\n      \"endTime\": \"2026-09-21T05:27:33.685762872Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv6ctdi2qdv7iqjg03ojai036om0fl6 still exists (deletion dispatched at 2026-09-21T05:03:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8c90086df972de01a5f3d9a1512bd148cfe020239facee7dcc2ca0e9fab90dcd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6092/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101882720345919488","occurred_at":"2026-09-21T03:54:26Z","failed_at":"e2e","signature_id":"4172f68b655680ae82c532494762946bf10e35ddc18ec7805954c2dd6cce8f2a","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001b762b8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-p2kds8\" in resourcegroup=\"rg-np-skip-minor-p2kds8-kwnbdqwdrkgw\" to finish deleting: GET https://rp.j5919488.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a77f009-3282-4c90-b388-0c23ff705330\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a77f009-3282-4c90-b388-0c23ff705330\",\n      \"name\": \"7a77f009-3282-4c90-b388-0c23ff705330\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T05:22:35.642993718Z\",\n      \"endTime\": \"2026-09-21T05:46:36.119766351Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv6cqf3r4salflml7bgo7m9e9pifk5q still exists (deletion dispatched at 2026-09-21T05:22:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001b762b8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-p2kds8\" in resourcegroup=\"rg-np-skip-minor-p2kds8-kwnbdqwdrkgw\" to finish deleting: GET https://rp.j5919488.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a77f009-3282-4c90-b388-0c23ff705330\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a77f009-3282-4c90-b388-0c23ff705330\",\n      \"name\": \"7a77f009-3282-4c90-b388-0c23ff705330\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T05:22:35.642993718Z\",\n      \"endTime\": \"2026-09-21T05:46:36.119766351Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv6cqf3r4salflml7bgo7m9e9pifk5q still exists (deletion dispatched at 2026-09-21T05:22:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101869717533757440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6225,"pr_state":"closed","pr_sha":"bbc846705098ffc40ca1855a736a470fc39488a1","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T03:02:46Z","started_at":"2026-09-21T03:02:46Z","completed_at":"2026-09-21T05:27:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T05:40:26.742451023Z","timing_metadata_checked_at":"2026-09-21T05:40:26.742451023Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T05:40:26.746298509Z","region_metadata_checked_at":"2026-09-21T05:40:26.746298509Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"c384fd711fb32b1e8e5c9983496cf93262f6e97b69f1844709edea3f913c80a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101869717533757440","occurred_at":"2026-09-21T03:02:46Z","failed_at":"e2e","signature_id":"146af559e2a271b5ea464bf8f71730a490f8a3b83a01c26dc8a3ed75481b34f0","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0017520c0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-hgtfk2fjv887\" to finish deleting: GET https://rp.j3757440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/720f938b-9af6-4f34-8272-bfaa20e19b67\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/720f938b-9af6-4f34-8272-bfaa20e19b67\",\n      \"name\": \"720f938b-9af6-4f34-8272-bfaa20e19b67\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T04:43:32.763976272Z\",\n      \"endTime\": \"2026-09-21T05:07:32.660675468Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv5qkbmlqjisdapk5l4guvgsq71b6j9 still exists (deletion dispatched at 2026-09-21T04:43:34Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0017520c0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-hgtfk2fjv887\" to finish deleting: GET https://rp.j3757440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/720f938b-9af6-4f34-8272-bfaa20e19b67\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/720f938b-9af6-4f34-8272-bfaa20e19b67\",\n      \"name\": \"720f938b-9af6-4f34-8272-bfaa20e19b67\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T04:43:32.763976272Z\",\n      \"endTime\": \"2026-09-21T05:07:32.660675468Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv5qkbmlqjisdapk5l4guvgsq71b6j9 still exists (deletion dispatched at 2026-09-21T04:43:34Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cc5b68dcd4dbb5ebd37355e5c71f0d7c646fa803c9ad73d83279898f5390ab1e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101869717533757440","occurred_at":"2026-09-21T03:02:46Z","failed_at":"alert","signature_id":"23d9580f300ff6d06018d0dda01666fa0578e116b96963bb783b71e03c866998","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-21T05:09:57Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j3757440-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-9k98z\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-9k98z (namespace arobit) on cluster ci01-j3757440-svc has averaged 124.6% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"f27cf1ad2d1e8108b85e41825262629d4b247340489e0f32af0aeb476d99ccd0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101869717533757440","occurred_at":"2026-09-21T03:02:46Z","failed_at":"alert","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T04:18:24Z\n  Ended:       2026-09-21T04:26:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3757440-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101869655269314560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7032,"pr_state":"closed","pr_sha":"c17bf1b8be7e620f98afeeb00dde43c86e8ebf20","final_merged_sha":"c17bf1b8be7e620f98afeeb00dde43c86e8ebf20","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T03:02:31Z","started_at":"2026-09-21T03:02:31Z","completed_at":"2026-09-21T04:32:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T05:18:46.69221101Z","timing_metadata_checked_at":"2026-09-21T05:18:46.69221101Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T05:18:46.696269713Z","region_metadata_checked_at":"2026-09-21T05:18:46.696269713Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"22fc02a76a5ee68e16cd7a7019b6fcd9968139618c32caf0b752d9c5cf4b707d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101869655269314560","occurred_at":"2026-09-21T03:02:31Z","failed_at":"alert","signature_id":"4477b1f779c5bccd585b79feb312432e19c78148cb9f08ab0e113c962c25f927","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 23 time(s) (3 unknown, 20 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T04:06:07Z\n  Ended:       2026-09-21T04:14:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9314560-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv5p6n0mkeb9fggti8c47tsk9keuq7l-agg-cluster\", pod=\"etcd-2\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sv5p6n0mkeb9fggti8c47tsk9keuq7l-agg-cluster/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-21T04:07:06Z\n  Ended:       2026-09-21T04:14:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9314560-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv5ortu9h2o8a77c86l3q2dojqo18s5-cilium-cluster\", pod=\"etcd-2\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sv5ortu9h2o8a77c86l3q2dojqo18s5-cilium-cluster/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-21T04:07:06Z\n  Ended:       2026-09-21T04:14:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9314560-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv5ortu9h2o8a77c86l3q2dojqo18s5-cilium-cluster\", pod=\"ignition-server-proxy-6ccf8c6444-24qfp\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sv5ortu9h2o8a77c86l3q2dojqo18s5-cilium-cluster/ignition-server-proxy-6ccf8c6444-24qfp has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101858124586553344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7036,"pr_state":"open","pr_sha":"5b4ed2bbcfd71a8855166afdecb608abdd6337cd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-21T02:16:42Z","started_at":"2026-09-21T02:16:42Z","completed_at":"2026-09-21T04:46:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T04:57:55.502791315Z","timing_metadata_checked_at":"2026-09-21T04:57:55.502791315Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T04:57:55.506407903Z","region_metadata_checked_at":"2026-09-21T04:57:55.506407903Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"10f5960866fe7699ec1c71da6605b43b3aad7431150d1ef4f86d60f14b2bb563","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101858124586553344","occurred_at":"2026-09-21T02:16:42Z","failed_at":"alert","signature_id":"4a784a26cd5b9756cf32ee23b4e60f7502e1f49a67a8d806a6d942ebdc472898","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 25 time(s) (2 unknown, 23 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T03:29:18Z\n  Ended:       2026-09-21T03:35:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6553344-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv56oltrdjme8j8laq16dpf3uvs62o8-x7r1k4p1e9k2r7a\", pod=\"cluster-network-operator-6c8887dd6f-5lxrp\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sv56oltrdjme8j8laq16dpf3uvs62o8-x7r1k4p1e9k2r7a/cluster-network-operator-6c8887dd6f-5lxrp has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-21T03:29:18Z\n  Ended:       2026-09-21T03:35:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6553344-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv56oltrdjme8j8laq16dpf3uvs62o8-x7r1k4p1e9k2r7a\", pod=\"ingress-operator-8459995bdf-h2f6l\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sv56oltrdjme8j8laq16dpf3uvs62o8-x7r1k4p1e9k2r7a/ingress-operator-8459995bdf-h2f6l has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"39a100f0d6ded2871a169a74b37a38c2118561494faa13648a1cc362d6d045f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101858124586553344","occurred_at":"2026-09-21T02:16:42Z","failed_at":"e2e","signature_id":"2c6296e3743e16c66f337f77fbddc58d9c8e2bd3e9ea1395f3adf63e9193e081","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001262168\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-dg-minor-rjsgfk\" in resourcegroup=\"rg-np-dg-minor-rjsgfk-9nzc9d562x6f\" to finish deleting: GET https://rp.j6553344.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9df6f16d-c44c-4665-bdf7-ecc7ff48b0f1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9df6f16d-c44c-4665-bdf7-ecc7ff48b0f1\",\n      \"name\": \"9df6f16d-c44c-4665-bdf7-ecc7ff48b0f1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T04:02:34.970644715Z\",\n      \"endTime\": \"2026-09-21T04:26:34.932726541Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv56omlf3347d47cl7ud3tk5o4s3a5q still exists (deletion dispatched at 2026-09-21T04:02:36Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001262168\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-dg-minor-rjsgfk\" in resourcegroup=\"rg-np-dg-minor-rjsgfk-9nzc9d562x6f\" to finish deleting: GET https://rp.j6553344.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9df6f16d-c44c-4665-bdf7-ecc7ff48b0f1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9df6f16d-c44c-4665-bdf7-ecc7ff48b0f1\",\n      \"name\": \"9df6f16d-c44c-4665-bdf7-ecc7ff48b0f1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T04:02:34.970644715Z\",\n      \"endTime\": \"2026-09-21T04:26:34.932726541Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv56omlf3347d47cl7ud3tk5o4s3a5q still exists (deletion dispatched at 2026-09-21T04:02:36Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6c8d3dcee8bc2c2cea91fbe76044a3dc74fc69daeac5b6322e3001d3ed9aac63","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101858124586553344","occurred_at":"2026-09-21T02:16:42Z","failed_at":"alert","signature_id":"59a65791b820d0a83953e3ed774047a0ae8d8779721276890da3197b50c4fb1d","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T03:53:36Z\n  Ended:       2026-09-21T04:08:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j6553344-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-6654f9cd75-pdqg6\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container kube-applier in pod kube-applier-6654f9cd75-pdqg6 (namespace kube-applier) on cluster ci01-j6553344-mgmt-1 has averaged 127.9% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-21T04:08:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j6553344-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-zx4xr\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-zx4xr (namespace arobit) on cluster ci01-j6553344-svc has averaged 126.8% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"e038eaa3bc0f84e592d9019ad92af64da696f27c2280b6d80604caadaf9459f0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101858124586553344","occurred_at":"2026-09-21T02:16:42Z","failed_at":"e2e","signature_id":"83b8e307efb2a908b207f3944abb0af36810467a632607f91624cdc945e2f3ee","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0010c2048\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-ng5tlp namespace are ready: failed to list pods in cilium-connectivity-check-ng5tlp namespace: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0010c2048\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-ng5tlp namespace are ready: failed to list pods in cilium-connectivity-check-ng5tlp namespace: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"e98663639effaacefa51fbb0f1d942616409a68a2e65da2bc867d3413b29ed08","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101858124586553344","occurred_at":"2026-09-21T02:16:42Z","failed_at":"e2e","signature_id":"0a2725b528c7209245a6c25fde003e4f8afada01d93f16ca7a3eee78e58045a2","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a6e168\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-845fl6wcvv94\" to finish deleting: GET https://rp.j6553344.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49c2cd99-164f-4496-86fb-c9c3403119d3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49c2cd99-164f-4496-86fb-c9c3403119d3\",\n      \"name\": \"49c2cd99-164f-4496-86fb-c9c3403119d3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T04:04:12.402255803Z\",\n      \"endTime\": \"2026-09-21T04:28:14.941695422Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv56qer13raejonhso57m958847e3qd still exists (deletion dispatched at 2026-09-21T04:04:14Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a6e168\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-845fl6wcvv94\" to finish deleting: GET https://rp.j6553344.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49c2cd99-164f-4496-86fb-c9c3403119d3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49c2cd99-164f-4496-86fb-c9c3403119d3\",\n      \"name\": \"49c2cd99-164f-4496-86fb-c9c3403119d3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T04:04:12.402255803Z\",\n      \"endTime\": \"2026-09-21T04:28:14.941695422Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv56qer13raejonhso57m958847e3qd still exists (deletion dispatched at 2026-09-21T04:04:14Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f9d68a7ea5be0e7809f7892291d2520885cfa9240cc144573ec914e32db8b6a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101858124586553344","occurred_at":"2026-09-21T02:16:42Z","failed_at":"alert","signature_id":"bbd3796eeb2ce791f30f3ec9695ea183e01029185ff0b1e971064a1b0229b720","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T03:17:45Z\n  Ended:       2026-09-21T03:51:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6553344-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6553344-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101853992668631040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6965,"pr_state":"closed","pr_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","final_merged_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T02:00:17Z","started_at":"2026-09-21T02:00:17Z","completed_at":"2026-09-21T04:15:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T04:26:29.386325064Z","timing_metadata_checked_at":"2026-09-21T04:26:29.386325064Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T04:26:29.389859851Z","region_metadata_checked_at":"2026-09-21T04:26:29.389859851Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"1d65b5d4bcf46edff34fb15cd254c676c69471cfe0f86a6615f059b108c040fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101853992668631040","occurred_at":"2026-09-21T02:00:17Z","failed_at":"e2e","signature_id":"ba532ab63715cfb85c685aa82f87796a84fd4aeb0462fb57bf48d07b9ea3853e","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00019f1c0\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-whfs4ngcqkr6\" to finish creating: GET https://rp.j8631040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/32525d6c-adbe-46db-ae4c-1e7b3b4cc233\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/32525d6c-adbe-46db-ae4c-1e7b3b4cc233\",\n      \"name\": \"32525d6c-adbe-46db-ae4c-1e7b3b4cc233\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:56:41.463150178Z\",\n      \"endTime\": \"2026-09-21T03:15:45.880215659Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00019f1c0\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-whfs4ngcqkr6\" to finish creating: GET https://rp.j8631040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/32525d6c-adbe-46db-ae4c-1e7b3b4cc233\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/32525d6c-adbe-46db-ae4c-1e7b3b4cc233\",\n      \"name\": \"32525d6c-adbe-46db-ae4c-1e7b3b4cc233\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:56:41.463150178Z\",\n      \"endTime\": \"2026-09-21T03:15:45.880215659Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"26dd10ff56da16affe1646aa85dec2f6979b87e252be2032ecc475e2c0a3baf3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101853992668631040","occurred_at":"2026-09-21T02:00:17Z","failed_at":"alert","signature_id":"81c439e5187a9d2934fe5bc20c61e52ed7c14b52c68df5b71a4836732d2dd783","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T03:24:40Z\n  Ended:       2026-09-21T03:38:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j8631040-svc\", component=\"frontend\", method=\"delete\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: More than 5% of frontend requests for delete /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j8631040-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}},{"row_id":"3fe82d0a40f365d134eb4913f9f1c1199c05c2687066af8707be4515ef4841fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101853992668631040","occurred_at":"2026-09-21T02:00:17Z","failed_at":"e2e","signature_id":"eb558ad9f2969bbcee6dfc5462cfefb3eaf744042053db96c583bea8dcc34878","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001580180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-lxc77m55522r\" to finish deleting: GET https://rp.j8631040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/32e2fa2a-d3d1-4865-b374-bb569224e9c3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/32e2fa2a-d3d1-4865-b374-bb569224e9c3\",\n      \"name\": \"32e2fa2a-d3d1-4865-b374-bb569224e9c3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T03:35:12.943000379Z\",\n      \"endTime\": \"2026-09-21T03:59:19.515561871Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv4pa8efr32u58nn479kf76m9d9kiov still exists (deletion dispatched at 2026-09-21T03:35:14Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001580180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-lxc77m55522r\" to finish deleting: GET https://rp.j8631040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/32e2fa2a-d3d1-4865-b374-bb569224e9c3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/32e2fa2a-d3d1-4865-b374-bb569224e9c3\",\n      \"name\": \"32e2fa2a-d3d1-4865-b374-bb569224e9c3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T03:35:12.943000379Z\",\n      \"endTime\": \"2026-09-21T03:59:19.515561871Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv4pa8efr32u58nn479kf76m9d9kiov still exists (deletion dispatched at 2026-09-21T03:35:14Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"42ee7631a748edbc05f33105efbce4441d95e2aad7dc48e998b699fe7c485ce9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101853992668631040","occurred_at":"2026-09-21T02:00:17Z","failed_at":"alert","signature_id":"b62f417ca2c5b1d6f88766222a5a157adc1bc0977d205edbc8c8baaca542db7d","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-21T03:54:57Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j8631040-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-m8gww\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-m8gww (namespace arobit) on cluster ci01-j8631040-svc has averaged 126.4% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7000,"pr_state":"closed","pr_sha":"e253ee7d7a59b02821f6339251ae15272721a63a","final_merged_sha":"e253ee7d7a59b02821f6339251ae15272721a63a","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T01:27:47Z","started_at":"2026-09-21T01:27:47Z","completed_at":"2026-09-21T03:45:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T03:55:43.800292249Z","timing_metadata_checked_at":"2026-09-21T03:55:43.800292249Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T03:55:43.804044537Z","region_metadata_checked_at":"2026-09-21T03:55:43.804044537Z"},"failed_at":["alert","e2e"],"failed_test_count":35,"occurrences":[{"row_id":"1b2ce21e872cba5a28843f181a65a389feedde1f17ae853cf3fa3b50048dc0fd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"96efdfeeb6070975b002e7c22c584e5a2b48bb653d226d1fa80cae7b27e1f16c","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00039ad20\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-q77sclqppp8q\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/20e25e07-c318-4b08-97c4-ee5e37c62d18\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/20e25e07-c318-4b08-97c4-ee5e37c62d18\",\n      \"name\": \"20e25e07-c318-4b08-97c4-ee5e37c62d18\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:45:59.100829855Z\",\n      \"endTime\": \"2026-09-21T03:15:04.812000965Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00039ad20\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-q77sclqppp8q\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/20e25e07-c318-4b08-97c4-ee5e37c62d18\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/20e25e07-c318-4b08-97c4-ee5e37c62d18\",\n      \"name\": \"20e25e07-c318-4b08-97c4-ee5e37c62d18\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:45:59.100829855Z\",\n      \"endTime\": \"2026-09-21T03:15:04.812000965Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1ba48c7b835a83e2c12dde87c821d4b8ceccd67df94310abae35a6cd091f0773","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"alert","signature_id":"3afcb0d9c9b46c7a9d0d57daf1a329110332585c824dbc79ee2008b67c2b174c","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T02:23:19Z\n  Ended:       2026-09-21T02:56:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j0544384-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j0544384-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"1d456ae429ba0b681cb051bdc228b9aaecb6feff09ed2d01c197686e4967c8c3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"ac52e380da4f27edb962f934785271597bc3ab8fa756e8897c620f29184fb960","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-mw6g5w\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001074720\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-mw6g5w: failed waiting for cluster=\"cp-ystream-upgrade-4-22-mw6g5w\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-7p4p8m4bt8mr\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a15d6a87-b85b-4e26-8932-a985ee962808\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a15d6a87-b85b-4e26-8932-a985ee962808\",\n      \"name\": \"a15d6a87-b85b-4e26-8932-a985ee962808\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:19.312896782Z\",\n      \"endTime\": \"2026-09-21T02:49:24.718568525Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-mw6g5w\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001074720\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-mw6g5w: failed waiting for cluster=\"cp-ystream-upgrade-4-22-mw6g5w\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-7p4p8m4bt8mr\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a15d6a87-b85b-4e26-8932-a985ee962808\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a15d6a87-b85b-4e26-8932-a985ee962808\",\n      \"name\": \"a15d6a87-b85b-4e26-8932-a985ee962808\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:19.312896782Z\",\n      \"endTime\": \"2026-09-21T02:49:24.718568525Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1fc1055eda98197714db6daea4c70f5da04cfe7f990e3891ca7c03b810d678f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"b2a29405053d7ce7694228479830d5216bc3a66458d76fc55218280e5418a0d0","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00035c6a0\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-fsm6ssdtjrd4\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/35ce6139-ac52-4039-b2e8-ddfd78835def\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/35ce6139-ac52-4039-b2e8-ddfd78835def\",\n      \"name\": \"35ce6139-ac52-4039-b2e8-ddfd78835def\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:19:49.466429352Z\",\n      \"endTime\": \"2026-09-21T02:48:55.452915135Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00035c6a0\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-fsm6ssdtjrd4\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/35ce6139-ac52-4039-b2e8-ddfd78835def\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/35ce6139-ac52-4039-b2e8-ddfd78835def\",\n      \"name\": \"35ce6139-ac52-4039-b2e8-ddfd78835def\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:19:49.466429352Z\",\n      \"endTime\": \"2026-09-21T02:48:55.452915135Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2559d006dd2881417fa29679f13ac61bd0eae713a14f790be463cd3828bf5c35","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"101cdce494d77d883d20d0eca54b7cb3633a3bef2ee73fd65d718f3585bbc1e4","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000944ce0\u003e: \n    failed to create HCP cluster np-downgrade-pg2f5g: failed waiting for cluster=\"np-downgrade-pg2f5g\" in resourcegroup=\"rg-np-downgrade-pg2f5g-x9t2rw964n4f\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4eb55b38-0373-4fce-befc-4d5851c7acc1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4eb55b38-0373-4fce-befc-4d5851c7acc1\",\n      \"name\": \"4eb55b38-0373-4fce-befc-4d5851c7acc1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:57.450123548Z\",\n      \"endTime\": \"2026-09-21T02:50:04.742128516Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000944ce0\u003e: \n    failed to create HCP cluster np-downgrade-pg2f5g: failed waiting for cluster=\"np-downgrade-pg2f5g\" in resourcegroup=\"rg-np-downgrade-pg2f5g-x9t2rw964n4f\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4eb55b38-0373-4fce-befc-4d5851c7acc1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4eb55b38-0373-4fce-befc-4d5851c7acc1\",\n      \"name\": \"4eb55b38-0373-4fce-befc-4d5851c7acc1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:57.450123548Z\",\n      \"endTime\": \"2026-09-21T02:50:04.742128516Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"32286688528bf2f2710ed454358f818133a59fad21e2c711a7342596668c72a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"83b077868da1af182cc41501a464f76776ad84dc44c11c6d2c403508163e831f","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011824c0\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-jcgg2jdbr6vw\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d3f17135-c725-4d66-b9cc-3263c8fba1a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d3f17135-c725-4d66-b9cc-3263c8fba1a2\",\n      \"name\": \"d3f17135-c725-4d66-b9cc-3263c8fba1a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:05.542631515Z\",\n      \"endTime\": \"2026-09-21T02:49:15.060122068Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011824c0\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-jcgg2jdbr6vw\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d3f17135-c725-4d66-b9cc-3263c8fba1a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d3f17135-c725-4d66-b9cc-3263c8fba1a2\",\n      \"name\": \"d3f17135-c725-4d66-b9cc-3263c8fba1a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:05.542631515Z\",\n      \"endTime\": \"2026-09-21T02:49:15.060122068Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"331d57f0498fa01ecdec05cd105f4f6b199e7c7d04bf7e678b367b0000aad0f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"6d0f960c5fdfdac6246b26b12b4175a2d9547452e5fee5317d4042e0e72fb4fc","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-pkp29r\" with version 4.22.13 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ffa440\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-pkp29r: failed waiting for cluster=\"cluster-zstream-4-22-pkp29r\" in resourcegroup=\"rg-zstream-upgrade-4-22-r2ntg795gnmd\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/10b076e2-f237-477d-b8db-9aec501bd0c7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/10b076e2-f237-477d-b8db-9aec501bd0c7\",\n      \"name\": \"10b076e2-f237-477d-b8db-9aec501bd0c7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:09.91357324Z\",\n      \"endTime\": \"2026-09-21T02:49:15.038726553Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-pkp29r\" with version 4.22.13 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ffa440\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-pkp29r: failed waiting for cluster=\"cluster-zstream-4-22-pkp29r\" in resourcegroup=\"rg-zstream-upgrade-4-22-r2ntg795gnmd\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/10b076e2-f237-477d-b8db-9aec501bd0c7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/10b076e2-f237-477d-b8db-9aec501bd0c7\",\n      \"name\": \"10b076e2-f237-477d-b8db-9aec501bd0c7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:09.91357324Z\",\n      \"endTime\": \"2026-09-21T02:49:15.038726553Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"355f527fe9f1269cf32cef7eeb7edff4256d15c7bddd291104f5fdbdb697a22a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"b9c63f623ffbcf0723ad68689e9ac0c775fa35e374b0606bd4f2a6edde72dce4","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--zm8bnqqhzsdd/cluster-hs-pre-hn4mh5 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b76b00\u003e: \n    failed to create HCP cluster cluster-hs-pre-hn4mh5: failed waiting for cluster=\"cluster-hs-pre-hn4mh5\" in resourcegroup=\"rg-hs-pre--zm8bnqqhzsdd\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/71af242e-526f-4585-9792-8ef889ff83e2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/71af242e-526f-4585-9792-8ef889ff83e2\",\n      \"name\": \"71af242e-526f-4585-9792-8ef889ff83e2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:09.775008999Z\",\n      \"endTime\": \"2026-09-21T02:49:14.818339561Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--zm8bnqqhzsdd/cluster-hs-pre-hn4mh5 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b76b00\u003e: \n    failed to create HCP cluster cluster-hs-pre-hn4mh5: failed waiting for cluster=\"cluster-hs-pre-hn4mh5\" in resourcegroup=\"rg-hs-pre--zm8bnqqhzsdd\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/71af242e-526f-4585-9792-8ef889ff83e2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/71af242e-526f-4585-9792-8ef889ff83e2\",\n      \"name\": \"71af242e-526f-4585-9792-8ef889ff83e2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:09.775008999Z\",\n      \"endTime\": \"2026-09-21T02:49:14.818339561Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3d95c58f4477ef0bd79c8e7a29f5b00b17f8f6285686b56af085756729254dd6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"bb5222936c947084dd170a35640be6d2ee22e0528f6daf36bada3db488de527f","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a8e240\u003e: \n    failed to create HCP cluster ea-listx2x86h: failed waiting for cluster=\"ea-listx2x86h\" in resourcegroup=\"ea-list-qd5tbqt9tkmw\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1d20ee7c-bb44-4630-95eb-26d2b196a06b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1d20ee7c-bb44-4630-95eb-26d2b196a06b\",\n      \"name\": \"1d20ee7c-bb44-4630-95eb-26d2b196a06b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:19.723545575Z\",\n      \"endTime\": \"2026-09-21T02:49:24.931588169Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a8e240\u003e: \n    failed to create HCP cluster ea-listx2x86h: failed waiting for cluster=\"ea-listx2x86h\" in resourcegroup=\"ea-list-qd5tbqt9tkmw\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1d20ee7c-bb44-4630-95eb-26d2b196a06b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1d20ee7c-bb44-4630-95eb-26d2b196a06b\",\n      \"name\": \"1d20ee7c-bb44-4630-95eb-26d2b196a06b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:19.723545575Z\",\n      \"endTime\": \"2026-09-21T02:49:24.931588169Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4362c66a51ec4556e0b13b28b56de03ab34b3076444fbfad439e395f9f2fc8c5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"445ee0601ffc1c563ea5fcd1ce58554b10e0cedf7590d0c2d884bb7a9983ad23","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c22980\u003e: \n    failed to create HCP cluster np-dg-minor-8dtwk7: failed waiting for cluster=\"np-dg-minor-8dtwk7\" in resourcegroup=\"rg-np-dg-minor-8dtwk7-wtlk2m8f2x7r\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fa08ff27-6abc-4eb2-9ed0-96ff9ce01ceb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fa08ff27-6abc-4eb2-9ed0-96ff9ce01ceb\",\n      \"name\": \"fa08ff27-6abc-4eb2-9ed0-96ff9ce01ceb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:07.466322123Z\",\n      \"endTime\": \"2026-09-21T02:49:15.047480993Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c22980\u003e: \n    failed to create HCP cluster np-dg-minor-8dtwk7: failed waiting for cluster=\"np-dg-minor-8dtwk7\" in resourcegroup=\"rg-np-dg-minor-8dtwk7-wtlk2m8f2x7r\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fa08ff27-6abc-4eb2-9ed0-96ff9ce01ceb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fa08ff27-6abc-4eb2-9ed0-96ff9ce01ceb\",\n      \"name\": \"fa08ff27-6abc-4eb2-9ed0-96ff9ce01ceb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:07.466322123Z\",\n      \"endTime\": \"2026-09-21T02:49:15.047480993Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"48598cb137248eabf5b27f2ded85d7362d147679253821d3183881e2cd89d95d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"cb7a95f81e6a4d2a68a733316e6739b9bf217de93fc5b210e74e99af512d90a4","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000874ee0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-dc8zxcnkw6xp\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b834056a-477e-4acb-a336-e7f172accb1c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b834056a-477e-4acb-a336-e7f172accb1c\",\n      \"name\": \"b834056a-477e-4acb-a336-e7f172accb1c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:14.375608638Z\",\n      \"endTime\": \"2026-09-21T02:49:14.98508687Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000874ee0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-dc8zxcnkw6xp\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b834056a-477e-4acb-a336-e7f172accb1c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b834056a-477e-4acb-a336-e7f172accb1c\",\n      \"name\": \"b834056a-477e-4acb-a336-e7f172accb1c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:14.375608638Z\",\n      \"endTime\": \"2026-09-21T02:49:14.98508687Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"495fcd8f67c76c964d2852e0a3e00485fbd8b1e783fc3e1cb6a4e65c5e91b31f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"83be0779e5c371ba66cee41003d41d1e4a94f36c0d9b77f574825ad8cdd51ea2","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001806360\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-97tblrgptv5d\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/119b1664-1dee-4c15-ad13-fc4e8e1ce8a4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/119b1664-1dee-4c15-ad13-fc4e8e1ce8a4\",\n      \"name\": \"119b1664-1dee-4c15-ad13-fc4e8e1ce8a4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:19:49.411376599Z\",\n      \"endTime\": \"2026-09-21T02:48:55.388071135Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001806360\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-97tblrgptv5d\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/119b1664-1dee-4c15-ad13-fc4e8e1ce8a4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/119b1664-1dee-4c15-ad13-fc4e8e1ce8a4\",\n      \"name\": \"119b1664-1dee-4c15-ad13-fc4e8e1ce8a4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:19:49.411376599Z\",\n      \"endTime\": \"2026-09-21T02:48:55.388071135Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4b53ad31034c4204ecb808a6796e36d01b31e5bb1511645caa72d712ca1376e5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"a15bb2ccd0901c3b476165088079dee1326b484696c7a412368c0abe97e696f0","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010a66a0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-g6d4svbs662q\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4d4ab1f4-0125-42c9-a511-09faf3065ed1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4d4ab1f4-0125-42c9-a511-09faf3065ed1\",\n      \"name\": \"4d4ab1f4-0125-42c9-a511-09faf3065ed1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:10.539041185Z\",\n      \"endTime\": \"2026-09-21T02:49:15.028073737Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010a66a0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-g6d4svbs662q\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4d4ab1f4-0125-42c9-a511-09faf3065ed1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4d4ab1f4-0125-42c9-a511-09faf3065ed1\",\n      \"name\": \"4d4ab1f4-0125-42c9-a511-09faf3065ed1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:10.539041185Z\",\n      \"endTime\": \"2026-09-21T02:49:15.028073737Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4c4a428c9ad74f380e9a3b6cfaf6fbeaa1cb6826e95a8386e606ffefe31d3823","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"d59cab2f620467077951c14df50b35f4e99cc6111239d251304f90209493e231","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-9cjrkc with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ffe940\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-9cjrkc: failed waiting for cluster=\"np-version-upgrade-cluster-9cjrkc\" in resourcegroup=\"rg-np-version-upgrade-9cjrkc-j5hx9lgs6lcv\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7a173705-b4b8-4a96-98ff-414002170922\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7a173705-b4b8-4a96-98ff-414002170922\",\n      \"name\": \"7a173705-b4b8-4a96-98ff-414002170922\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:05.871026696Z\",\n      \"endTime\": \"2026-09-21T02:49:14.703066158Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-9cjrkc with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ffe940\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-9cjrkc: failed waiting for cluster=\"np-version-upgrade-cluster-9cjrkc\" in resourcegroup=\"rg-np-version-upgrade-9cjrkc-j5hx9lgs6lcv\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7a173705-b4b8-4a96-98ff-414002170922\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7a173705-b4b8-4a96-98ff-414002170922\",\n      \"name\": \"7a173705-b4b8-4a96-98ff-414002170922\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:05.871026696Z\",\n      \"endTime\": \"2026-09-21T02:49:14.703066158Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"50b627dad1326cc88ea7b8060cea7217ba88e3acb08425e42f13e0a18bc4dadd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"998a2530a32bfdbab7bb61026e80849c50959079f93409c7baef1a69e6ce8f55","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010065c0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-9ckq2tpgs8nf\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0122bbc2-d7cc-47d9-a62b-2d02f56c70bf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0122bbc2-d7cc-47d9-a62b-2d02f56c70bf\",\n      \"name\": \"0122bbc2-d7cc-47d9-a62b-2d02f56c70bf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:18.752619331Z\",\n      \"endTime\": \"2026-09-21T02:49:24.949727548Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010065c0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-9ckq2tpgs8nf\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0122bbc2-d7cc-47d9-a62b-2d02f56c70bf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0122bbc2-d7cc-47d9-a62b-2d02f56c70bf\",\n      \"name\": \"0122bbc2-d7cc-47d9-a62b-2d02f56c70bf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:18.752619331Z\",\n      \"endTime\": \"2026-09-21T02:49:24.949727548Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"54181780a0e829919cab07ddb494d37a4f6f9d41bd9185d9542e20656cbce3ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"a4139e19e1756c804e8b75c885064c44d2813ba52683007423e82f2dd8e15f4c","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00109e240\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-nphgn2d545mx\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7c3c2670-1993-4126-995d-b0d3798263d2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7c3c2670-1993-4126-995d-b0d3798263d2\",\n      \"name\": \"7c3c2670-1993-4126-995d-b0d3798263d2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:19:59.794374029Z\",\n      \"endTime\": \"2026-09-21T02:49:04.905263613Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00109e240\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-nphgn2d545mx\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7c3c2670-1993-4126-995d-b0d3798263d2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7c3c2670-1993-4126-995d-b0d3798263d2\",\n      \"name\": \"7c3c2670-1993-4126-995d-b0d3798263d2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:19:59.794374029Z\",\n      \"endTime\": \"2026-09-21T02:49:04.905263613Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6056135e14b2ec27ebe73377e52610c29944bc1191162b3d33f0707070c3a1d6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"401bdeb7e2e5f6ff8be5adc09f7dd4fe0535fef399a47ebba371fe4c28f3324f","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-4nlwmq\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ca4500\u003e: \n    failed to create HCP cluster np-noedge-4nlwmq: failed waiting for cluster=\"np-noedge-4nlwmq\" in resourcegroup=\"rg-np-noedge-4nlwmq-xw9ltmrm2vfg\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9260158f-f236-49d0-a451-fb478173f867\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9260158f-f236-49d0-a451-fb478173f867\",\n      \"name\": \"9260158f-f236-49d0-a451-fb478173f867\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:57.302956845Z\",\n      \"endTime\": \"2026-09-21T02:50:05.153563844Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-4nlwmq\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ca4500\u003e: \n    failed to create HCP cluster np-noedge-4nlwmq: failed waiting for cluster=\"np-noedge-4nlwmq\" in resourcegroup=\"rg-np-noedge-4nlwmq-xw9ltmrm2vfg\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9260158f-f236-49d0-a451-fb478173f867\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9260158f-f236-49d0-a451-fb478173f867\",\n      \"name\": \"9260158f-f236-49d0-a451-fb478173f867\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:57.302956845Z\",\n      \"endTime\": \"2026-09-21T02:50:05.153563844Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7e9471022a3cd649abf0f49e06fadc030bcfe0662436e44ad96a5aeb01064ce3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"583190e317f030b6d4dcc78e0f8e9549bf5e175870bbe9ed92dca4fd5c761be0","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c925c0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-qjqm4qhjzsqp\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/484d2c87-fa1e-417e-b724-b101df392854\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/484d2c87-fa1e-417e-b724-b101df392854\",\n      \"name\": \"484d2c87-fa1e-417e-b724-b101df392854\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:21:42.140301649Z\",\n      \"endTime\": \"2026-09-21T02:50:44.925161016Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c925c0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-qjqm4qhjzsqp\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/484d2c87-fa1e-417e-b724-b101df392854\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/484d2c87-fa1e-417e-b724-b101df392854\",\n      \"name\": \"484d2c87-fa1e-417e-b724-b101df392854\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:21:42.140301649Z\",\n      \"endTime\": \"2026-09-21T02:50:44.925161016Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8b1839fe6f250967bba9e68c0d10394641aab6843cc59af8333d895205973a37","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"bf2a7da9caad7da5ae2e85c89ed44488df414516533be98e57a5f74072293524","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001032320\u003e: \n    failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-4xflb9p5l8j9\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a6d3cd92-f445-4feb-be31-165c16892821\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a6d3cd92-f445-4feb-be31-165c16892821\",\n      \"name\": \"a6d3cd92-f445-4feb-be31-165c16892821\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:19:49.859289926Z\",\n      \"endTime\": \"2026-09-21T02:48:55.208760693Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001032320\u003e: \n    failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-4xflb9p5l8j9\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a6d3cd92-f445-4feb-be31-165c16892821\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a6d3cd92-f445-4feb-be31-165c16892821\",\n      \"name\": \"a6d3cd92-f445-4feb-be31-165c16892821\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:19:49.859289926Z\",\n      \"endTime\": \"2026-09-21T02:48:55.208760693Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8ea616e6e26437b5a33b28c7197eb1eb432c4961bd5e1c4990a21ce1a0da6ee7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"30359cbb5d8dab177085f627742624bf4aff26ff5ff2bf62dcef875ee400238a","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fe62e0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-b7gtsbk76cn9\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/627ebc1b-6227-4297-a3c2-d1736e67605d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/627ebc1b-6227-4297-a3c2-d1736e67605d\",\n      \"name\": \"627ebc1b-6227-4297-a3c2-d1736e67605d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:21:40.675582383Z\",\n      \"endTime\": \"2026-09-21T02:50:44.756111292Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fe62e0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-b7gtsbk76cn9\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/627ebc1b-6227-4297-a3c2-d1736e67605d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/627ebc1b-6227-4297-a3c2-d1736e67605d\",\n      \"name\": \"627ebc1b-6227-4297-a3c2-d1736e67605d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:21:40.675582383Z\",\n      \"endTime\": \"2026-09-21T02:50:44.756111292Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"90530b2ccf323754fe0a12feac33b578b01038f50333c48464c6c6d6c3f50272","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"68f9273ee5f51f662683ce559f799fdc9db250b242e9d9858c3377319443b335","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fca600\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-n4ms8fvhjcw9\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/45e74e5d-d71c-4f37-b2b4-6b9907f7d9b3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/45e74e5d-d71c-4f37-b2b4-6b9907f7d9b3\",\n      \"name\": \"45e74e5d-d71c-4f37-b2b4-6b9907f7d9b3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:10.074821606Z\",\n      \"endTime\": \"2026-09-21T02:49:15.055559478Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fca600\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-n4ms8fvhjcw9\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/45e74e5d-d71c-4f37-b2b4-6b9907f7d9b3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/45e74e5d-d71c-4f37-b2b4-6b9907f7d9b3\",\n      \"name\": \"45e74e5d-d71c-4f37-b2b4-6b9907f7d9b3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:10.074821606Z\",\n      \"endTime\": \"2026-09-21T02:49:15.055559478Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9778cf8bd3111a332decaebaac533e7a65d8f707efea5c00c0d54974a8c6155c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"f06eadc7025693b52366ed66c3fea2961df4ee597cfe9c923d08d52d4da02532","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b1d580\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-fwccm9qcgxcs\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a56df295-26b2-4770-afde-352f0d7293c2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a56df295-26b2-4770-afde-352f0d7293c2\",\n      \"name\": \"a56df295-26b2-4770-afde-352f0d7293c2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:06.295516429Z\",\n      \"endTime\": \"2026-09-21T02:49:15.078703959Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b1d580\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-fwccm9qcgxcs\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a56df295-26b2-4770-afde-352f0d7293c2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a56df295-26b2-4770-afde-352f0d7293c2\",\n      \"name\": \"a56df295-26b2-4770-afde-352f0d7293c2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:06.295516429Z\",\n      \"endTime\": \"2026-09-21T02:49:15.078703959Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9858006968159ea21a11af93c3e26cd9cbff21648f20c84e3125e7a992c9ecc7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"alert","signature_id":"6154aca91fdbe6f7a740067ce60dc27b028619981066db25335cdb4192c93be8","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-21T02:59:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0544384-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.205:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0544384/providers/microsoft.monitor/accounts/services-j0544384\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-hv4kl\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-21T02:59:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0544384-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.205:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0544384/providers/microsoft.monitor/accounts/services-j0544384\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-hv4kl\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired"}},{"row_id":"988b52d763677bff922c9c1f91f88dd82fc72d62fe3794465b71568dec8a7872","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"f45e06d3a5cf3624317eeb24a6885b33e5e99d697e44a2edd7412805fe743274","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001388138\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-w8msz29ss5lr\" to finish deleting: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d5a9ac1b-e225-4681-ba4a-098250305696\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d5a9ac1b-e225-4681-ba4a-098250305696\",\n      \"name\": \"d5a9ac1b-e225-4681-ba4a-098250305696\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T03:06:12.814249025Z\",\n      \"endTime\": \"2026-09-21T03:30:17.065460546Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv4cou1kjtqros9fe5sm297hg48ffaj still exists (deletion dispatched at 2026-09-21T03:06:13Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001388138\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-w8msz29ss5lr\" to finish deleting: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d5a9ac1b-e225-4681-ba4a-098250305696\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d5a9ac1b-e225-4681-ba4a-098250305696\",\n      \"name\": \"d5a9ac1b-e225-4681-ba4a-098250305696\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T03:06:12.814249025Z\",\n      \"endTime\": \"2026-09-21T03:30:17.065460546Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv4cou1kjtqros9fe5sm297hg48ffaj still exists (deletion dispatched at 2026-09-21T03:06:13Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"999f5b6726576bdc6972717b0b96a33288abdb452ae7e98331395bf0e3285098","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"90122d534d247281d565dbed025294c56e77c1b1af3fb2917f98d0948a4fcc01","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-pk2sm9smw6hc/cluster-candidate-4-21-xrjq2k should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000afc360\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-xrjq2k: failed waiting for cluster=\"cluster-candidate-4-21-xrjq2k\" in resourcegroup=\"rg-candidate-4-21-pk2sm9smw6hc\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/80ba225e-1c08-43fc-923e-524380934131\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/80ba225e-1c08-43fc-923e-524380934131\",\n      \"name\": \"80ba225e-1c08-43fc-923e-524380934131\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:10.044098563Z\",\n      \"endTime\": \"2026-09-21T02:49:15.018100902Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-pk2sm9smw6hc/cluster-candidate-4-21-xrjq2k should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000afc360\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-xrjq2k: failed waiting for cluster=\"cluster-candidate-4-21-xrjq2k\" in resourcegroup=\"rg-candidate-4-21-pk2sm9smw6hc\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/80ba225e-1c08-43fc-923e-524380934131\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/80ba225e-1c08-43fc-923e-524380934131\",\n      \"name\": \"80ba225e-1c08-43fc-923e-524380934131\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:10.044098563Z\",\n      \"endTime\": \"2026-09-21T02:49:15.018100902Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a6b5d7aa1538448442c18d1f0279041752b131a3e378aa7ab6c6400e48351f23","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"4c7dd5bd4c1fc1d1abc223bc954b26926543610053daa1e89b007495d40b2e7e","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b5d880\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-k6zrn9g5mfk5\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96ecbd6d-a9d0-4d00-b4d6-d51e987f2c1f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96ecbd6d-a9d0-4d00-b4d6-d51e987f2c1f\",\n      \"name\": \"96ecbd6d-a9d0-4d00-b4d6-d51e987f2c1f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:42.832729875Z\",\n      \"endTime\": \"2026-09-21T02:49:45.338859237Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b5d880\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-k6zrn9g5mfk5\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96ecbd6d-a9d0-4d00-b4d6-d51e987f2c1f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96ecbd6d-a9d0-4d00-b4d6-d51e987f2c1f\",\n      \"name\": \"96ecbd6d-a9d0-4d00-b4d6-d51e987f2c1f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:42.832729875Z\",\n      \"endTime\": \"2026-09-21T02:49:45.338859237Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b43a8b762d016b1c2d54edd1f14d36ccd81a9d2860edfb5f994d867b40f8ef52","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"3c52ca174905abfb7f02e429c357c86685f31a7746602bb280efc1d9536b6a08","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a64c80\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-lzbpqchwtpjq\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/50782cfe-84f8-43a4-836f-7dfb3ca52030\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/50782cfe-84f8-43a4-836f-7dfb3ca52030\",\n      \"name\": \"50782cfe-84f8-43a4-836f-7dfb3ca52030\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:21:02.502165407Z\",\n      \"endTime\": \"2026-09-21T02:50:05.126194832Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a64c80\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-lzbpqchwtpjq\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/50782cfe-84f8-43a4-836f-7dfb3ca52030\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/50782cfe-84f8-43a4-836f-7dfb3ca52030\",\n      \"name\": \"50782cfe-84f8-43a4-836f-7dfb3ca52030\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:21:02.502165407Z\",\n      \"endTime\": \"2026-09-21T02:50:05.126194832Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c166197f0f89963bd79b1a9803f14dee9ed64723fdb4f970ceeb7c626c2848c1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"alert","signature_id":"e42d229f3a2ec2f54d131cb62d52d064b20f3baed44dfb084036ae491c555913","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T02:53:04Z\n  Ended:       2026-09-21T03:00:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0544384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.205:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0544384/providers/microsoft.monitor/accounts/services-j0544384\", namespace=\"klusterlet-2sv4cqnh3tg4c1kjg6jvt3g7omcg6sqq\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-hv4kl\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sv4cqnh3tg4c1kjg6jvt3g7omcg6sqq/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-21T02:53:05Z\n  Ended:       2026-09-21T02:59:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0544384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.205:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0544384/providers/microsoft.monitor/accounts/services-j0544384\", namespace=\"klusterlet-2sv4cmlktjpm9h9svdn985c6grh4d055\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-hv4kl\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sv4cmlktjpm9h9svdn985c6grh4d055/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}},{"row_id":"c3b7a51ea3e815a30cd33266775655309e9bbfc0bca25d247b118f4923893aa8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"alert","signature_id":"daecb21a101cfdd07bfaeee76cd4ee81d2c911fa7ae4325a728c3c1c94bcd389","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-21T02:21:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0544384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-54f7d49479-4gspp\", region=\"centralus\", severity=\"warning\"\n  Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-54f7d49479-4gspp has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"c6d9f2590ae2e00ed2d9cc3c1d6a93c624d105be2ac9845a076d8d51a0886dab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"9949f03a8a3602b3e73c9bf334697336a9345e18d49d34e8707e6bdfc7251484","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:83]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b00940\u003e: \n    failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-4cnsgj6kmd6w\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/816a6153-8f4d-4bba-99b1-4eb2b3b7ea67\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/816a6153-8f4d-4bba-99b1-4eb2b3b7ea67\",\n      \"name\": \"816a6153-8f4d-4bba-99b1-4eb2b3b7ea67\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:25.455164762Z\",\n      \"endTime\": \"2026-09-21T02:49:34.903464846Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:83]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b00940\u003e: \n    failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-4cnsgj6kmd6w\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/816a6153-8f4d-4bba-99b1-4eb2b3b7ea67\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/816a6153-8f4d-4bba-99b1-4eb2b3b7ea67\",\n      \"name\": \"816a6153-8f4d-4bba-99b1-4eb2b3b7ea67\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:25.455164762Z\",\n      \"endTime\": \"2026-09-21T02:49:34.903464846Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ce7f5c9d1c0e7f6f563c550a17c54e28f7f025330404e9a0e07dae4153af6617","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"d98eca1d04fea272d4ea34cd3375fc5ba03bc4c8d72f4ea270e5db83f3343194","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"564f3e52406178650115a5099e6f58ea4f33418fc9d8b3ed9ad25dd72ff83dfd","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-l5v4pc with version 4.20.38\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f906c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-l5v4pc: failed waiting for cluster=\"np-version-upgrade-cluster-l5v4pc\" in resourcegroup=\"rg-np-version-upgrade-l5v4pc-npdppbd24ml9\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/935a2043-8104-4feb-b285-dcfd941f8e66\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/935a2043-8104-4feb-b285-dcfd941f8e66\",\n      \"name\": \"935a2043-8104-4feb-b285-dcfd941f8e66\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:25.736836775Z\",\n      \"endTime\": \"2026-09-21T02:49:34.945024032Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-l5v4pc with version 4.20.38\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f906c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-l5v4pc: failed waiting for cluster=\"np-version-upgrade-cluster-l5v4pc\" in resourcegroup=\"rg-np-version-upgrade-l5v4pc-npdppbd24ml9\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/935a2043-8104-4feb-b285-dcfd941f8e66\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/935a2043-8104-4feb-b285-dcfd941f8e66\",\n      \"name\": \"935a2043-8104-4feb-b285-dcfd941f8e66\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:25.736836775Z\",\n      \"endTime\": \"2026-09-21T02:49:34.945024032Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e0afcf39d241701687e0bbe1495fb4c8633125726245907a15f17175c5f3cd3e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"7fc9fccc7dc3326cd5189c2310966c7923bcf86c7ba85f77fdaaf5d2989ab411","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001b08e0\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-ksztg8zv6f7k\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e62a071a-70bc-4232-8c2b-6ab3745925da\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e62a071a-70bc-4232-8c2b-6ab3745925da\",\n      \"name\": \"e62a071a-70bc-4232-8c2b-6ab3745925da\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:19:59.20808577Z\",\n      \"endTime\": \"2026-09-21T02:49:05.005743646Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001b08e0\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-ksztg8zv6f7k\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e62a071a-70bc-4232-8c2b-6ab3745925da\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e62a071a-70bc-4232-8c2b-6ab3745925da\",\n      \"name\": \"e62a071a-70bc-4232-8c2b-6ab3745925da\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:19:59.20808577Z\",\n      \"endTime\": \"2026-09-21T02:49:05.005743646Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e671ff57982a3bf0d40bdbd3fc42ecf610eb0f2c210aaec22d2b1a0d8a1d8712","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"bbc0ada550aeb8ef55b59f58426467c3c5437b52079022c76ea642780d30a836","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b969e0\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-vgq2xbt8l9l5\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/575e5cb0-c9fe-489b-a249-1e12298af52a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/575e5cb0-c9fe-489b-a249-1e12298af52a\",\n      \"name\": \"575e5cb0-c9fe-489b-a249-1e12298af52a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:45:58.611416768Z\",\n      \"endTime\": \"2026-09-21T03:15:04.814025973Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b969e0\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-vgq2xbt8l9l5\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/575e5cb0-c9fe-489b-a249-1e12298af52a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/575e5cb0-c9fe-489b-a249-1e12298af52a\",\n      \"name\": \"575e5cb0-c9fe-489b-a249-1e12298af52a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:45:58.611416768Z\",\n      \"endTime\": \"2026-09-21T03:15:04.814025973Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e6b26007e6a3f88b6e3ca48c7e700461358590ed76019458d3349a3612f9ac5a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101845814870544384","occurred_at":"2026-09-21T01:27:47Z","failed_at":"e2e","signature_id":"83cc7fad540436e557df3ca8771763651675253c907752745ae416c84c24e057","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-jfjw8ln9sczz/cluster-candidate-4-20-7klkb7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011e2380\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-7klkb7: failed waiting for cluster=\"cluster-candidate-4-20-7klkb7\" in resourcegroup=\"rg-candidate-4-20-jfjw8ln9sczz\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a761c674-5d49-4779-86da-db54c2cc6504\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a761c674-5d49-4779-86da-db54c2cc6504\",\n      \"name\": \"a761c674-5d49-4779-86da-db54c2cc6504\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:10.825968118Z\",\n      \"endTime\": \"2026-09-21T02:49:15.011005165Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-jfjw8ln9sczz/cluster-candidate-4-20-7klkb7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011e2380\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-7klkb7: failed waiting for cluster=\"cluster-candidate-4-20-7klkb7\" in resourcegroup=\"rg-candidate-4-20-jfjw8ln9sczz\" to finish creating: GET https://rp.j0544384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a761c674-5d49-4779-86da-db54c2cc6504\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a761c674-5d49-4779-86da-db54c2cc6504\",\n      \"name\": \"a761c674-5d49-4779-86da-db54c2cc6504\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T02:20:10.825968118Z\",\n      \"endTime\": \"2026-09-21T02:49:15.011005165Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":35,"failure_pattern_ids":["4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":35,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7035/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101824255497867264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7035,"pr_state":"closed","pr_sha":"18faff8c97058d15283061028cf5dd67da4bc619","final_merged_sha":"18faff8c97058d15283061028cf5dd67da4bc619","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T00:01:33Z","started_at":"2026-09-21T00:01:33Z","completed_at":"2026-09-21T00:11:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T00:28:40.948363842Z","timing_metadata_checked_at":"2026-09-21T00:28:40.948363842Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-21T00:28:40.951805531Z","region_metadata_checked_at":"2026-09-21T01:31:06.030692186Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"345677e54fadac1d8a24277e9faefae7b497fe880792487318ea102499e6a1ac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7035/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101824255497867264","occurred_at":"2026-09-21T00:01:33Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}}]},{"environment":"int","summary":{"total_runs":5,"failed_runs":1,"runs_with_occurrences":1,"runs_with_failure_pattern_matches":1,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":4,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2102173520803926016","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-21T23:09:58Z","started_at":"2026-09-21T23:09:58Z","completed_at":"2026-09-22T01:31:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T01:46:14.154555534Z","timing_metadata_checked_at":"2026-09-22T01:46:14.154555534Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T01:46:14.158529122Z","region_metadata_checked_at":"2026-09-22T01:46:14.158529122Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2102082659676065792","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-21T17:00:41Z","started_at":"2026-09-21T17:00:41Z","completed_at":"2026-09-21T19:28:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T19:40:39.407577328Z","timing_metadata_checked_at":"2026-09-21T19:40:39.407577328Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T19:40:39.410050421Z","region_metadata_checked_at":"2026-09-21T19:40:39.410050421Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2101994817666945024","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-21T11:19:52Z","started_at":"2026-09-21T11:19:52Z","completed_at":"2026-09-21T13:40:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T13:52:38.389578674Z","timing_metadata_checked_at":"2026-09-21T13:52:38.389578674Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T13:52:38.393091962Z","region_metadata_checked_at":"2026-09-21T13:52:38.393091962Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2101944127254958080","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T07:58:26Z","started_at":"2026-09-21T07:58:26Z","completed_at":"2026-09-21T10:14:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T10:25:00.550619291Z","timing_metadata_checked_at":"2026-09-21T10:25:00.550619291Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T10:25:00.553673281Z","region_metadata_checked_at":"2026-09-21T10:25:00.553673281Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"e6d252c9731d818ba7444d8a006d207d1ec52eba8eaf217e38a5bc4b34ccf1ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2101944127254958080","occurred_at":"2026-09-21T07:58:26Z","failed_at":"e2e","signature_id":"8d140b5c9e70d3dec87d293bf19cc617ab277a59f4ca19b6e5cead44ffb558f6","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.003s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000b78000\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.003s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000b78000\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2463341f6ec16eab820eae75ad2ab931f4e65db1d48f95432f3ae076654a6478","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2463341f6ec16eab820eae75ad2ab931f4e65db1d48f95432f3ae076654a6478"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2101890552306864128","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-21T04:17:23Z","started_at":"2026-09-21T04:17:23Z","completed_at":"2026-09-21T06:43:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T06:53:43.540837248Z","timing_metadata_checked_at":"2026-09-21T06:53:43.540837248Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T06:53:43.543744737Z","region_metadata_checked_at":"2026-09-21T06:53:43.543744737Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]},{"environment":"prod","summary":{"total_runs":2,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102171179442769920","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T23:00:40Z","started_at":"2026-09-21T23:00:40Z","completed_at":"2026-09-22T00:58:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T01:04:18.684484013Z","timing_metadata_checked_at":"2026-09-22T01:04:18.684484013Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T01:04:18.686968805Z","region_metadata_checked_at":"2026-09-22T01:04:18.686968805Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"052692eece497fe9b65078701f7d784187b88c0afcc5911632c536ee9854f0fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102171179442769920","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"94cf9b9a0502b5068c9238768b8aace2dae96cc40dbe7aa6c4fcd5f29f95c663","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00157b410\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-gmj7znm6krq2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00157b410\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-gmj7znm6krq2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"0b8684009cd7a49d9acabb94a97c0442c3c9e2712aea807eb74c5bbf2eddde2d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102171179442769920","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"437d8c76cf25382ab68ccb3d1dfc352505d64eb1166c75bb4020b96761195fe7","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e87aa0\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-jppjzb\" resourcegroup=\"rg-np-dg-minor-jppjzb-mqbgnthqv6nw\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-jppjzb-mqbgnthqv6nw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-jppjzb/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e87aa0\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-jppjzb\" resourcegroup=\"rg-np-dg-minor-jppjzb-mqbgnthqv6nw\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-jppjzb-mqbgnthqv6nw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-jppjzb/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"0be6286fa371268a7cce11854e7ee8ce82961e91d3dceaf547efa0911df0026b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102171179442769920","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"9f7db5b69df1f2dc135087f8d38f0fe6f2043d6e8eb169e0fe72692ead0d203d","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0010069f0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-fsdhfh67dtp2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0010069f0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-fsdhfh67dtp2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"1717bddd2f8865ce2869fa6c01b443dbc9c65438906b935cea676cc3371976b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102171179442769920","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"55667292d9d8eeeed6366e69c7cbab5e2a7d8906031f5af72403715d79303536","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-5-1-9dm6vxblqqwm/cluster-nightly-5-1-2mzlb4 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e2a620\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-2mzlb4: failed waiting for cluster=\"cluster-nightly-5-1-2mzlb4\" in resourcegroup=\"rg-nightly-5-1-9dm6vxblqqwm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e5936c3-5c07-4597-b323-aa8c4ac28e70\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e5936c3-5c07-4597-b323-aa8c4ac28e70\",\n      \"name\": \"1e5936c3-5c07-4597-b323-aa8c4ac28e70\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T23:36:42.177277415Z\",\n      \"endTime\": \"2026-09-22T00:05:42.40340744Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-5-1-9dm6vxblqqwm/cluster-nightly-5-1-2mzlb4 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e2a620\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-2mzlb4: failed waiting for cluster=\"cluster-nightly-5-1-2mzlb4\" in resourcegroup=\"rg-nightly-5-1-9dm6vxblqqwm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e5936c3-5c07-4597-b323-aa8c4ac28e70\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e5936c3-5c07-4597-b323-aa8c4ac28e70\",\n      \"name\": \"1e5936c3-5c07-4597-b323-aa8c4ac28e70\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T23:36:42.177277415Z\",\n      \"endTime\": \"2026-09-22T00:05:42.40340744Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4528b434af694f8025e1b1474e4ef626ed5489dcd675669ca845a20091661f31","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1c92436a1e3146eb1f89d9ad737d9be192d6cbe5bc50ed0a710d4458d6b54893","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102171179442769920","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"ccfc6d74e1d4632e4c75cd1a817e3edf960e048681d88c62fc1dea51de4a0818","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0009249c0\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-21-111728, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0009249c0\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-21-111728, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"43a14e221aa822007da1cef22ac936dcf9bde999082056dae8bf1e8357ed5f71","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102171179442769920","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"5c11fbed65a0c6bbf0d4fad9d1914159225c8b41459d81bd839e59628e6d85ca","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-wlqfz9\" for cluster \"cluster-nightly-4-20-wlqfz9\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00096e3a0\u003e: \n    failed to create NodePool np-wlqfz9: failed starting nodepool creation \"np-wlqfz9\" for cluster \"cluster-nightly-4-20-wlqfz9\" in resourcegroup=\"rg-nightly-4-20-7q72mxd46fc9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-7q72mxd46fc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-wlqfz9/nodePools/np-wlqfz9\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-wlqfz9\" for cluster \"cluster-nightly-4-20-wlqfz9\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00096e3a0\u003e: \n    failed to create NodePool np-wlqfz9: failed starting nodepool creation \"np-wlqfz9\" for cluster \"cluster-nightly-4-20-wlqfz9\" in resourcegroup=\"rg-nightly-4-20-7q72mxd46fc9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-7q72mxd46fc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-wlqfz9/nodePools/np-wlqfz9\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"482b20c5707c1a347993c1084cbf33323ebd22e6d458e8d43df484858943ab7b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102171179442769920","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"6f5854152632398eea399e230e43eae00f99e75d4775214b2b657966479c20db","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:274]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000a388d0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-hnrk7b4v2x6b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:274]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000a388d0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-hnrk7b4v2x6b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"65a17de5d3d8d1e1f99ecce9e5725b45702cd1bbf6c307d8b908ade8d5a95afc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102171179442769920","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"b0eaac4568056adc6173a1fa00e432aa6852e4ce888f67cd930cc490d060b229","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00062e820\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-rnt89sjcb8mz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-rnt89sjcb8mz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00062e820\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-rnt89sjcb8mz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-rnt89sjcb8mz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d6164cff5eccd3db4fabd19fbe08aa12f39c6b3d5c2202182c9f291029d16282","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.22\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"86ee36b5e0b138bcc5c7d0308589163b1cc978f32d63a597a5b4080c86ca2419","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102171179442769920","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"77f882940cb8e625ac686c0e0085dd6cc0a5dc7d95cfa2d311bc0cc60084de78","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f8fac0\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-xmffvj\" in resourcegroup=\"rg-np-version-upgrade-xmffvj-bddpsjrtf2pf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-xmffvj-bddpsjrtf2pf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-xmffvj/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f8fac0\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-xmffvj\" in resourcegroup=\"rg-np-version-upgrade-xmffvj-bddpsjrtf2pf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-xmffvj-bddpsjrtf2pf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-xmffvj/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"8bdc60bfb90d70c91653fc076bae7d70c4c12e828c08ed3f267bf7770bf44883","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102171179442769920","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"4fc5bb8b74f41a7b8c27c86f543d89ef4bb831c4aa3b468206417a08ba6ebbba","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc002e0bcb0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-5nj7xwd9d8lp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc002e0bcb0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-5nj7xwd9d8lp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a3f478578255e8a568c2ee0d5d1ea7f82c2c856a193bb8692b4db1304bf201c4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102171179442769920","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"16d7d69670ea28b84e5bd3dbbf9c3e77453975b69c364e436ac77a71ffb5ac53","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c586c0\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-5ldnts\" in resourcegroup=\"rg-np-skip-minor-5ldnts-pnzgxgcmlk5s\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-5ldnts-pnzgxgcmlk5s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-5ldnts/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c586c0\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-5ldnts\" in resourcegroup=\"rg-np-skip-minor-5ldnts-pnzgxgcmlk5s\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-5ldnts-pnzgxgcmlk5s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-5ldnts/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"f5ee210aa6b31ad81bffa86c0285548e41fc02c9e6cbfb20264d9daec6ad6ddb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102171179442769920","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"88de2c5eaca5de3966d86532b905f732a4f20880b980eeb6e76aa5beccfaf268","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0028a88a0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-rng5tfgqsvfr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0028a88a0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-rng5tfgqsvfr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["4528b434af694f8025e1b1474e4ef626ed5489dcd675669ca845a20091661f31","495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","d6164cff5eccd3db4fabd19fbe08aa12f39c6b3d5c2202182c9f291029d16282"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2101854085815734272","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T02:00:39Z","started_at":"2026-09-21T02:00:39Z","completed_at":"2026-09-21T04:14:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T04:26:29.539916236Z","timing_metadata_checked_at":"2026-09-21T04:26:29.539916236Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T04:26:29.543634623Z","region_metadata_checked_at":"2026-09-21T04:26:29.543634623Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c606497942c6c412b5c06d4e9746389717d22bfa8e928369577450a37ddca598","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2101854085815734272","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"70878cd3674d1a051d369bfea35296f9681ecc3659c011e955f6a6d7ff0ca7b8","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00155c2e0\u003e: \n    failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-tnpj8h\" in resourcegroup=\"rg-np-version-upgrade-tnpj8h-m6skjqh8vcv8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/670619ae-29a5-476c-912b-af06c657c780\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/670619ae-29a5-476c-912b-af06c657c780\",\n      \"name\": \"670619ae-29a5-476c-912b-af06c657c780\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T03:17:13.234002267Z\",\n      \"endTime\": \"2026-09-21T03:36:17.606656785Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00155c2e0\u003e: \n    failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-tnpj8h\" in resourcegroup=\"rg-np-version-upgrade-tnpj8h-m6skjqh8vcv8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/670619ae-29a5-476c-912b-af06c657c780\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/670619ae-29a5-476c-912b-af06c657c780\",\n      \"name\": \"670619ae-29a5-476c-912b-af06c657c780\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T03:17:13.234002267Z\",\n      \"endTime\": \"2026-09-21T03:36:17.606656785Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]},{"environment":"stg","summary":{"total_runs":2,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T23:00:40Z","started_at":"2026-09-21T23:00:40Z","completed_at":"2026-09-21T23:16:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T23:19:41.983493779Z","timing_metadata_checked_at":"2026-09-21T23:19:41.983493779Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T23:19:41.986725069Z","region_metadata_checked_at":"2026-09-21T23:19:41.986725069Z"},"failed_at":["e2e"],"failed_test_count":52,"occurrences":[{"row_id":"0109478a9320d31bd6f1f5d4576dcce9c5e133a5432ce43f68efefc15676de00","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"db807efcdd39ea2cb52af865c0882f1d3fb3c2898d624a1e367c703af42611e7","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c842c0\u003e: \n    failed to create HCP cluster np-dg-minor-t9q787: failed starting cluster creation \"np-dg-minor-t9q787\" in resourcegroup=\"rg-np-dg-minor-t9q787-jzfhg2wbjhw6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-t9q787-jzfhg2wbjhw6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-t9q787\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c842c0\u003e: \n    failed to create HCP cluster np-dg-minor-t9q787: failed starting cluster creation \"np-dg-minor-t9q787\" in resourcegroup=\"rg-np-dg-minor-t9q787-jzfhg2wbjhw6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-t9q787-jzfhg2wbjhw6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-t9q787\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"02164bed11dfea489c7a68c1b4af14e53db8505304c5f9e65be7d3d5fb85591a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"a6486b2d616fdb4383f7884aeca7d65fad4062f747c2a3b0cd9757ca4438a9e2","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007e2240\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-kmrh4fx4qqfs\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-kmrh4fx4qqfs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007e2240\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-kmrh4fx4qqfs\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-kmrh4fx4qqfs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"0c71068d7121d99fbe406e20230dc2f8a3448d060ec8b178023ad397a1373a2c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"cfe7b17485f9acc5aebee5bd74c0b43354d83bcb4b267c6f70f6759c40a36c05","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-5-1-5vjt4b7ggcgc/cluster-nightly-5-1-kkscf5 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001446320\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-kkscf5: failed starting cluster creation \"cluster-nightly-5-1-kkscf5\" in resourcegroup=\"rg-nightly-5-1-5vjt4b7ggcgc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-1-5vjt4b7ggcgc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-5-1-kkscf5\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-5-1-5vjt4b7ggcgc/cluster-nightly-5-1-kkscf5 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001446320\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-kkscf5: failed starting cluster creation \"cluster-nightly-5-1-kkscf5\" in resourcegroup=\"rg-nightly-5-1-5vjt4b7ggcgc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-1-5vjt4b7ggcgc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-5-1-kkscf5\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"12d8b31c87687cf16be5b1be6479a434556f930a27ace0e1a67bbe4018581572","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"90ce15eea497022aaa61820d10beba710ac951a2f9f0397e033aeb84463cc781","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-rrxrmh\" at install version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b90100\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-rrxrmh: failed starting cluster creation \"cp-ystream-upgrade-4-21-rrxrmh\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-z28lrzs6psj6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-z28lrzs6psj6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-rrxrmh\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-rrxrmh\" at install version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b90100\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-rrxrmh: failed starting cluster creation \"cp-ystream-upgrade-4-21-rrxrmh\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-z28lrzs6psj6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-z28lrzs6psj6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-rrxrmh\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"1535efc99ed8a2833d8f7b7c50941ba2c29c6fa4dd457e54753f08d8ee036d5d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"ef820fcc6468435f50b9022babd3c9bb015a295b55d8e688452bedfbd1d7c4d8","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00158a0e0\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \"patch-tags-cluster\" in resourcegroup=\"patch-tags-w7hqz7mgfw74\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-w7hqz7mgfw74/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00158a0e0\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \"patch-tags-cluster\" in resourcegroup=\"patch-tags-w7hqz7mgfw74\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-w7hqz7mgfw74/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"185c323024ea558da65e3f347976582b2eaa56377846c6e8b2d42aa6e1eef884","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"070eda49f37b6789dd629760de53b01ffee3bba242dd00b8d0c301862f4957f7","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e2e180\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-hh7zrnvrpf5p\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-hh7zrnvrpf5p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e2e180\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-hh7zrnvrpf5p\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-hh7zrnvrpf5p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"1cf42aeecb7f426bc172ec2f9f1e9e9f3ce8fbbf5e26d5ba4abd0a3573992a06","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"bab743dc89a1cf85d69fba2507f8801e78481c902bae2a46c3f054e73e65fc5c","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00085e1a0\u003e: \n    failed to create HCP cluster ea-cluster: failed starting cluster creation \"ea-cluster\" in resourcegroup=\"external-auth-cluster-zgq4s984h4qj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-zgq4s984h4qj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00085e1a0\u003e: \n    failed to create HCP cluster ea-cluster: failed starting cluster creation \"ea-cluster\" in resourcegroup=\"external-auth-cluster-zgq4s984h4qj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-zgq4s984h4qj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"1f203c5b3182f511e5ca7b64da1b4d9b5aeea24348aea46bddfd1c5935c369c6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"76d95bd8807af30492cc05984c2913d54e7d509b85d70cabae6845b2ad48a543","test_name":"Customer should be able to update node pool labels and taints","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a84120\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed starting cluster creation \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-6fhrg9ql25mh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-6fhrg9ql25mh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a84120\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed starting cluster creation \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-6fhrg9ql25mh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-6fhrg9ql25mh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"24f16222004c856bbda291f39ff401d885c221713ef38db0cf50cb161c03b535","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"96f3def1f74208c3f2281d05d73db0d4011791b2ae17586ab6c80277c1311cb6","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dfa6a0\u003e: \n    failed starting cluster creation \"agg-cluster\" in resourcegroup=\"feature-aggregation-ch5mkxlx27sb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-ch5mkxlx27sb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dfa6a0\u003e: \n    failed starting cluster creation \"agg-cluster\" in resourcegroup=\"feature-aggregation-ch5mkxlx27sb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-ch5mkxlx27sb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"25eb300753484d034b79228afbda9c9cb411d2c647035407555fe4dcf8c1f7a1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"cbcb43059e58396eb0fd60ef4da5593a70dd90c1f9cd1811b69165d11ff09857","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006a6340\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-zjn2gfd7qtgh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-zjn2gfd7qtgh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006a6340\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-zjn2gfd7qtgh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-zjn2gfd7qtgh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"27776618802c5a48df8ece03c14a0fe7a41722d2415c81a257208a652716a4ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"2cc7446507fbd5237a91f1f64b2a72d1315104a08b25b0dc1fc3498c6168d6e9","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f78100\u003e: \n    failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-5t77lfxhsr6g\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-5t77lfxhsr6g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f78100\u003e: \n    failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-5t77lfxhsr6g\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-5t77lfxhsr6g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"2caf55d0933f076679f06de5d0d3245376a7f944f428a5a8165b894aa82b74d2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"4a9c3bb5bace633838b22dc46f4cc5d5f2f7c830cebfc8ec7b709be463dc4e9c","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000ded500\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-8vbmz2474dd2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000ded500\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-8vbmz2474dd2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"337f94f2e40a6bcc4bc666f24bd0f8f990ae991a2fe9a9f1d8b5333796ea95da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"32b461f2840f848dad902183508b8af65a076b98024a026dd0f5e6289191d092","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000920280\u003e: \n    failed starting cluster creation \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-f54tp4k8t8jc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-f54tp4k8t8jc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000920280\u003e: \n    failed starting cluster creation \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-f54tp4k8t8jc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-f54tp4k8t8jc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"338a286a7233705f15f6ea54fec35a4781b0f5d6cee1e3d0e09c086ae77a90f5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"b4a9c1191ec9bc882981699e33042718a557d6ea52125771fda9bff325abd6d3","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-p59hvv with version 4.21.0-0.nightly-multi-2026-09-21-183653\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c641c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-p59hvv: failed starting cluster creation \"np-version-upgrade-cluster-p59hvv\" in resourcegroup=\"rg-np-version-upgrade-p59hvv-r8k6rv29ncx8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-p59hvv-r8k6rv29ncx8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-p59hvv\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-p59hvv with version 4.21.0-0.nightly-multi-2026-09-21-183653\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c641c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-p59hvv: failed starting cluster creation \"np-version-upgrade-cluster-p59hvv\" in resourcegroup=\"rg-np-version-upgrade-p59hvv-r8k6rv29ncx8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-p59hvv-r8k6rv29ncx8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-p59hvv\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"35f5212d7d974e21ce8803e9c24729377a3592b858600839c9d4557c74e6c576","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"8fd0f930a2bbfdce0bd248eeecd47d8ee99dfc9219c4506e6fc4f67ba7b693ae","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-lfk9vt\" at install version 4.21.0-0.nightly-multi-2026-09-21-183653\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00072c580\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-lfk9vt: failed starting cluster creation \"cp-ystream-upgrade-4-22-lfk9vt\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-zb2j2v899x9w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-zb2j2v899x9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-lfk9vt\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-lfk9vt\" at install version 4.21.0-0.nightly-multi-2026-09-21-183653\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00072c580\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-lfk9vt: failed starting cluster creation \"cp-ystream-upgrade-4-22-lfk9vt\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-zb2j2v899x9w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-zb2j2v899x9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-lfk9vt\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"37ad588fb8de87fd01fd6a22bbd3944c0f06c70cdfc815823377c04dc819c669","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"42b9220a6d2f3d8609d44ad13ee37971174e8fb2961d57043d3da633e7a7c3c3","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bf6280\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-kmfvjnb249dn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-kmfvjnb249dn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bf6280\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-kmfvjnb249dn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-kmfvjnb249dn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"38de81a8e3346af63d10b68315d55b6fd0774c66e5c2a2106b781905939d2d1a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"6ec4d7a2ec4afbf570475d4ba858dd20d8705afefb49267a9b3acd41ded8acd6","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fe0160\u003e: \n    failed to create HCP cluster cilium-cl: failed starting cluster creation \"cilium-cl\" in resourcegroup=\"cni-cilium-5qlmkmmr6rbl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-5qlmkmmr6rbl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cl\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fe0160\u003e: \n    failed to create HCP cluster cilium-cl: failed starting cluster creation \"cilium-cl\" in resourcegroup=\"cni-cilium-5qlmkmmr6rbl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-5qlmkmmr6rbl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cl\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"3a0e86e03197c3d2385b774bdb16b353ed1b65945e2aebb32bc80779aa4f145b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"007c512b92fb41db26c7b8b7cba10b469d061b0c6325011486a52aca6754d1d1","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e60380\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed starting cluster creation \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-xkmrf4xtl8st\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-xkmrf4xtl8st/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-delete-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e60380\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed starting cluster creation \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-xkmrf4xtl8st\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-xkmrf4xtl8st/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-delete-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"44f41e44ec9657ece2bf3c2c6e38502d718bc7b79531fd4a2b3c62024ff3b2b5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"05356a41d90554df553c7f0f38bbf0bd6cc1dcccef464a6d7dc9ad4b9c635693","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014e02a0\u003e: \n    failed starting cluster creation \"private-kv-cluster\" in resourcegroup=\"private-keyvault-n8584dkfk984\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-n8584dkfk984/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014e02a0\u003e: \n    failed starting cluster creation \"private-kv-cluster\" in resourcegroup=\"private-keyvault-n8584dkfk984\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-n8584dkfk984/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"4618ef2b1226412ad35eac7b88b3df4ebc91a2e913e636698a99c9fed08be3a8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"04ff69991092d2e571fa36c84684cc3410c79a5d9eb981fb9c0c387f85fe6815","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0016ea260\u003e: \n    failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-7k6gmjrrhndn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-7k6gmjrrhndn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0016ea260\u003e: \n    failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-7k6gmjrrhndn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-7k6gmjrrhndn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"474567438256274d84f40f956257d1605d38b5be15a155749bdb71502c3519db","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"01ec605fef5f44749d3695cb0210cbd1ad2aece5d5d3116f698ae59589dab7ed","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fb0120\u003e: \n    failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-g886g4ln75lz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-g886g4ln75lz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fb0120\u003e: \n    failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-g886g4ln75lz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-g886g4ln75lz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"4a6dbd0e9bc3edee49f716488109bd81eed2bd328e70326d81d609c1d46a9737","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"eea3f0eff6cc39e455129f680d3dda340879f19915e5fa235c7ec789f5807dde","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-nlth6q\" at install version 4.22.0-0.nightly-multi-2026-09-17-071442\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f9c260\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-nlth6q: failed starting cluster creation \"cp-ystream-upgrade-5-0-nlth6q\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-2gl8z49jglqz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-2gl8z49jglqz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-nlth6q\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-nlth6q\" at install version 4.22.0-0.nightly-multi-2026-09-17-071442\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f9c260\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-nlth6q: failed starting cluster creation \"cp-ystream-upgrade-5-0-nlth6q\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-2gl8z49jglqz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-2gl8z49jglqz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-nlth6q\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"4d2f6d1543ed01a4fac85ee0ddb352c5f2f39ee0e082cd22a6d36d146c6ba461","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"323dbb31e75de34be6db3ccd1313e87d7070831d2e1337711e0669f7f29c7d56","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000860160\u003e: \n    failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-bvkt9h245gbn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-bvkt9h245gbn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000860160\u003e: \n    failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-bvkt9h245gbn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-bvkt9h245gbn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"5114f0c04bb4ad6fd545c5d021fe4ae0a48f4aca4eae9ebee2711e038f937006","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"20ef48caaf1176eebf84cb12c1a4891bb2cf23f60a13675d11bfa3f34949bf3a","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b98160\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-sxrtgbzc7t6s\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-sxrtgbzc7t6s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b98160\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-sxrtgbzc7t6s\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-sxrtgbzc7t6s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"5f22e5f3b57df055387448ac8ac768eb03f90149cb09de24d1c965c4b3789392","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"54afb469a617c6853f5631e8c297ce0352a6e1375e5c13424c35a038e15a6173","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d08680\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed starting cluster creation \"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-lnst8xpwqcxf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-lnst8xpwqcxf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d08680\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed starting cluster creation \"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-lnst8xpwqcxf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-lnst8xpwqcxf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"605a760711517540a8e6eec2cfa3ff34c57bddd4302d52092100d667a5b4a3ae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"94d78a6b6251cda73419322f9c6d0211b2f50fce4b08f291d1849bec523a8c63","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0015ace70\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0015ace70\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6d245f481a15bd17cb6e2fb0336ccedc1f5575a881a64af4bce77702c008ab88","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"782291c497da32be1bd058b0761ceceb4292a2523b2c10d7fd930b1503350ba1","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a4a3e0\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed starting cluster creation \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-cvxxzvs6w9mk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-cvxxzvs6w9mk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a4a3e0\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed starting cluster creation \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-cvxxzvs6w9mk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-cvxxzvs6w9mk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"738f41547d2881af12310f8fa6a52cb3c9e2832c97377499910c5a0e765ce15d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"d34e67530c71d13e42118cac91a0ed782082d4c06f31e0c2ef88aba6e75be0be","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000efa260\u003e: \n    failed starting cluster creation \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-28r7lqx8kvzb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-28r7lqx8kvzb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000efa260\u003e: \n    failed starting cluster creation \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-28r7lqx8kvzb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-28r7lqx8kvzb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7ac0925a26d83789fc14461be407e7e87710e91c368bab735cf7f5f17a497e4e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"a3e0b7636787c0ef984c068795fb6d5dc1b7fb99f8ba82aeda4c83c2408f625b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-5-0-nmzkm84phphp/cluster-nightly-5-0-l69np5 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c1e1c0\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-l69np5: failed starting cluster creation \"cluster-nightly-5-0-l69np5\" in resourcegroup=\"rg-nightly-5-0-nmzkm84phphp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-nmzkm84phphp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-5-0-l69np5\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-5-0-nmzkm84phphp/cluster-nightly-5-0-l69np5 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c1e1c0\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-l69np5: failed starting cluster creation \"cluster-nightly-5-0-l69np5\" in resourcegroup=\"rg-nightly-5-0-nmzkm84phphp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-nmzkm84phphp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-5-0-l69np5\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"8335f0e9cd16bd13f761da66c96f8e715b48b53be8db80db624ff632c31ed233","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"5abb8cb9efccabc37a81a8c7ffcef3d9e652b64f00be449ba3b6aab94140c179","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001020240\u003e: \n    failed to create HCP cluster ea-listvx4z9m: failed starting cluster creation \"ea-listvx4z9m\" in resourcegroup=\"ea-list-5vzvrbm6nxhr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-5vzvrbm6nxhr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listvx4z9m\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001020240\u003e: \n    failed to create HCP cluster ea-listvx4z9m: failed starting cluster creation \"ea-listvx4z9m\" in resourcegroup=\"ea-list-5vzvrbm6nxhr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-5vzvrbm6nxhr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listvx4z9m\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"85ca20777afa9941cc8ef3a725ccfa40d328a329dbebce4bff48b89ae77652b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"30ad796179e97685bd64f934b0c58800eeb356baf11245a707e9e807a965b6d1","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000886180\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-r5wk5vbh49g2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-r5wk5vbh49g2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000886180\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-r5wk5vbh49g2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-r5wk5vbh49g2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"9052cef0439da384168587f016d35ec378067f6ef7b3c82d91388cc0daad2aa7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"62e68c2e3ddfffdb0d648f1bc2a8b91ac0affa1217f1e02d9564b320d26ee95b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-21-5vc6fdpfzlqr/cluster-nightly-4-21-b6b98x should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fca120\u003e: \n    failed to create HCP cluster cluster-nightly-4-21-b6b98x: failed starting cluster creation \"cluster-nightly-4-21-b6b98x\" in resourcegroup=\"rg-nightly-4-21-5vc6fdpfzlqr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-21-5vc6fdpfzlqr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-21-b6b98x\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-21-5vc6fdpfzlqr/cluster-nightly-4-21-b6b98x should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fca120\u003e: \n    failed to create HCP cluster cluster-nightly-4-21-b6b98x: failed starting cluster creation \"cluster-nightly-4-21-b6b98x\" in resourcegroup=\"rg-nightly-4-21-5vc6fdpfzlqr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-21-5vc6fdpfzlqr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-21-b6b98x\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"9d751128cc3937f3544f1920776045073e26743d2d1e0d654593658c68c20525","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"d0644c92a0ae860eba87eb8ef1197cb97a255e81f8cd7eb54169060d998c77c7","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00118a120\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-nw5mkq5xpnmq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-nw5mkq5xpnmq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00118a120\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-nw5mkq5xpnmq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-nw5mkq5xpnmq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"a64fee9aa71e665e71882b4cc50972897759a68a8374e3ea14c427d25515a3c8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"96a7a73cd2a244277e9033553ca2a1ebd1c0f4ef060bdb1d9ddec7f3d0392a26","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001250120\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-vknqqk8nhd9b\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-vknqqk8nhd9b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001250120\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-vknqqk8nhd9b\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-vknqqk8nhd9b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"b765aa782d7bcefa8723c1aeb9b0b77d3be4e0041838a6c30bbb5c7ed8b1f14a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"aaccf4367525f05f9d32301b975f5e74e140dd3699834fdcb3da64567da15e96","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:105]: failed to begin creating HCP cluster \"admin-cred-lifecycle-52lb7m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d16160\u003e: \n    failed starting cluster creation \"admin-cred-lifecycle-52lb7m\" in resourcegroup=\"admin-credential-lifecycle-test-2vklc49tbbf2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-2vklc49tbbf2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-52lb7m\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:105]: failed to begin creating HCP cluster \"admin-cred-lifecycle-52lb7m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d16160\u003e: \n    failed starting cluster creation \"admin-cred-lifecycle-52lb7m\" in resourcegroup=\"admin-credential-lifecycle-test-2vklc49tbbf2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-2vklc49tbbf2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-52lb7m\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"bb46b2ea67f63e3671c68136314d622d3a7c47f8ba8b114bdbfb37c11418279f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"c3c5f609987f4505811b1b5894071f3c173ed96a31f8285179c2552e4d88698d","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00011a9c0\u003e: \n    failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-k7dzr8jpr2d6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-k7dzr8jpr2d6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00011a9c0\u003e: \n    failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-k7dzr8jpr2d6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-k7dzr8jpr2d6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"bbc9b2d599e9d96be093fd80bbca8f0049bf3bf265bcafa499bb445ec1e12108","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"166c9106429214aaa90ce8529f97b363b4246f3c21b6a42164b44cd5f4f02773","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00080c140\u003e: \n    failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-ckl79nms47n9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-ckl79nms47n9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00080c140\u003e: \n    failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-ckl79nms47n9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-ckl79nms47n9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"bc03a8d696717ebcf98c32910612c376125d5bd8517979050ffd083caa4f56b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"c5e2cf9a49c0c072cf7333a0f4a4647757cfd960f64ca31f757271670c3dfdf7","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-qqx2xv\" at install version 4.22.0-0.nightly-multi-2026-09-17-071442\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fe4260\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-qqx2xv: failed starting cluster creation \"cp-ystream-upgrade-4-23-qqx2xv\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-rstlbhm7524g\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-rstlbhm7524g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-qqx2xv\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-qqx2xv\" at install version 4.22.0-0.nightly-multi-2026-09-17-071442\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fe4260\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-qqx2xv: failed starting cluster creation \"cp-ystream-upgrade-4-23-qqx2xv\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-rstlbhm7524g\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-rstlbhm7524g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-qqx2xv\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"bd75594ff701ee7e7e763e2653a52a92ca8fad33fb719f7cced37535562525a4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"10845029b2a1572fc7bde32a7d479000bc712bec74eb3f791cafe7c464be6c7d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-23-vv9xz4mh8lbb/cluster-nightly-4-23-qrn7fh should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b50100\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-qrn7fh: failed starting cluster creation \"cluster-nightly-4-23-qrn7fh\" in resourcegroup=\"rg-nightly-4-23-vv9xz4mh8lbb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-23-vv9xz4mh8lbb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-23-qrn7fh\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-23-vv9xz4mh8lbb/cluster-nightly-4-23-qrn7fh should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b50100\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-qrn7fh: failed starting cluster creation \"cluster-nightly-4-23-qrn7fh\" in resourcegroup=\"rg-nightly-4-23-vv9xz4mh8lbb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-23-vv9xz4mh8lbb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-23-qrn7fh\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c16ad9dedf00f0993dbb21d5631710a0bf3422e6c5ebe8ec5955b7fe28477e78","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"562a2d519e66af356f0a55b719ace62fbfe94188aabbbb42e2604f4dd1f4371d","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011001a0\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-jkzbwtnr7w7r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-jkzbwtnr7w7r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011001a0\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-jkzbwtnr7w7r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-jkzbwtnr7w7r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c80ef5047a45e500088ea4aa7c04274d733a9ac5cd976f878340f0c766fb787b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"b6dae38f134ad0fe73cc5b18b282eb2d0056791b16cfa0b90a9e2dca8ee0b738","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006de300\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed starting cluster creation \"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-qzp79pctmgfm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-qzp79pctmgfm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/missing-mi-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006de300\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed starting cluster creation \"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-qzp79pctmgfm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-qzp79pctmgfm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/missing-mi-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c8a0336d8fd7d61d374635ec1488a6f84aea230138029c6bff7b611aac4cdbfb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"743de13ef41c54b04482907b96e72065920f3dc5c2454012006153000ed152cf","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fa6f60\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-pm7qzd4sxcrl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-pm7qzd4sxcrl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fa6f60\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-pm7qzd4sxcrl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-pm7qzd4sxcrl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ca98de24578da275d5b4f146633d051dba9a9802bff06c783a034d022aaf6d70","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"de5805a5bfcb89822a4e21a8d61ad4b83a6d58c57ef1365c3414d2657174c6c0","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-22-sbnqqnhg48vx/cluster-nightly-4-22-xbkwmj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00089c7a0\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-xbkwmj: failed starting cluster creation \"cluster-nightly-4-22-xbkwmj\" in resourcegroup=\"rg-nightly-4-22-sbnqqnhg48vx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-sbnqqnhg48vx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-22-xbkwmj\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-22-sbnqqnhg48vx/cluster-nightly-4-22-xbkwmj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00089c7a0\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-xbkwmj: failed starting cluster creation \"cluster-nightly-4-22-xbkwmj\" in resourcegroup=\"rg-nightly-4-22-sbnqqnhg48vx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-sbnqqnhg48vx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-22-xbkwmj\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d1ca18ff780e8599656966c7b2dc78afd9c57fd713e4497e855516f6a2349204","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"af71484bf23751887c03592a70a4af04882b5f4eefec75343ddd2bbf419c0e0c","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00077c3c0\u003e: \n    failed to create HCP cluster node-limit-cluster: failed starting cluster creation \"node-limit-cluster\" in resourcegroup=\"node-limit-test-2sm7kd2dzbxw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-2sm7kd2dzbxw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00077c3c0\u003e: \n    failed to create HCP cluster node-limit-cluster: failed starting cluster creation \"node-limit-cluster\" in resourcegroup=\"node-limit-test-2sm7kd2dzbxw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-2sm7kd2dzbxw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d500629fae260d6af324c7b206d68f734a10a2b9726406a4e88d1dc6ad840714","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"6bc0d2a4ce26c694af456165368fccb93dd1223090c05657453d43d5d6d0ea97","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000136280\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-6fsm4k5srrbc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-6fsm4k5srrbc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000136280\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-6fsm4k5srrbc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-6fsm4k5srrbc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d578d03417c7dba3f3fe4e1e83fa53b4390d73f94ea7e2466c764e46fae14729","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"4e4a97930cef04be0e85bb7020c3b938f7b2e28cfa9e80cdd00cedfe1584bcac","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-5dt8cb\" at install version 5.0.0-0.nightly-multi-2026-09-21-111728\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b9a9a0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-1-5dt8cb: failed starting cluster creation \"cp-ystream-upgrade-5-1-5dt8cb\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-4vzwsqsnhmp9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-4vzwsqsnhmp9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-5dt8cb\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-5dt8cb\" at install version 5.0.0-0.nightly-multi-2026-09-21-111728\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b9a9a0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-1-5dt8cb: failed starting cluster creation \"cp-ystream-upgrade-5-1-5dt8cb\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-4vzwsqsnhmp9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-4vzwsqsnhmp9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-5dt8cb\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"e06bf7cf7b321dfc7f4a263ca1d09d797145b499d4d83c2952f44b33e3c2fb25","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"96dcf90d11ae3668b98d4c3d96f484a0ad93e956f7f14451494461c5c9f688d2","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-20-fw2c5skj5f5s/cluster-nightly-4-20-9bqspv should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010da180\u003e: \n    failed to create HCP cluster cluster-nightly-4-20-9bqspv: failed starting cluster creation \"cluster-nightly-4-20-9bqspv\" in resourcegroup=\"rg-nightly-4-20-fw2c5skj5f5s\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-fw2c5skj5f5s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-9bqspv\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-20-fw2c5skj5f5s/cluster-nightly-4-20-9bqspv should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010da180\u003e: \n    failed to create HCP cluster cluster-nightly-4-20-9bqspv: failed starting cluster creation \"cluster-nightly-4-20-9bqspv\" in resourcegroup=\"rg-nightly-4-20-fw2c5skj5f5s\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-fw2c5skj5f5s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-9bqspv\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"e2003056fb5a04aa9b9fee5032e85f8aea50c5251f2c968750042bf39e74c7c2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"1eee6895140653b4792838366a7dc0904a43f605f78826c519de47cf4351d9a7","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00104c260\u003e: \n    failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-kwfmtmz69pkl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-kwfmtmz69pkl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00104c260\u003e: \n    failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-kwfmtmz69pkl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-kwfmtmz69pkl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"e21e96a7d58879e55df2bd01c14183aaf8e57e3124769d75a787aee9ba197b8f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"b8e758968b3b2d445a597bb984284cb3df68c3bcea3ce5229ac774e7b6c28e16","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001048660\u003e: \n    failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-tdbmcx8l2gp5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-tdbmcx8l2gp5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001048660\u003e: \n    failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-tdbmcx8l2gp5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-tdbmcx8l2gp5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"e557ef9822a7215ace5f9bab136b8de546726694c6886b810a60651876f79339","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"afa45ddbde78f8dc858ef2693d6c3018004788e448a2081a96912085c960a116","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015942a0\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed starting cluster creation \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-d9n7pv8k2jh9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-d9n7pv8k2jh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015942a0\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed starting cluster creation \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-d9n7pv8k2jh9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-d9n7pv8k2jh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f1dff8c69430981da5538dcaad19959d2a201a91cd2598262446b9618a5857bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"1f842a9dbaaa9ce437db94a6d4b835a644c1c23915affa8e892648ccd8c01165","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001056280\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed starting cluster creation \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-kw2lcjsr974r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-kw2lcjsr974r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cx-rg-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001056280\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed starting cluster creation \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-kw2lcjsr974r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-kw2lcjsr974r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cx-rg-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"fc3f2f96698763a279ed43b1c9f81ee85664db30b4642c4b5623596cbe141cce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102171179497295872","occurred_at":"2026-09-21T23:00:40Z","failed_at":"e2e","signature_id":"bc2dbf7238c5f9054aa6efbae957b4367adff5149647259472f2e37fd888a8aa","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-sl9rc2 at version 4.22.0-0.nightly-multi-2026-09-17-071442\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0017f01e0\u003e: \n    failed to create HCP cluster np-skip-minor-sl9rc2: failed starting cluster creation \"np-skip-minor-sl9rc2\" in resourcegroup=\"rg-np-skip-minor-sl9rc2-6hzgpt7xvb74\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-sl9rc2-6hzgpt7xvb74/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-sl9rc2\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-sl9rc2 at version 4.22.0-0.nightly-multi-2026-09-17-071442\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0017f01e0\u003e: \n    failed to create HCP cluster np-skip-minor-sl9rc2: failed starting cluster creation \"np-skip-minor-sl9rc2\" in resourcegroup=\"rg-np-skip-minor-sl9rc2-6hzgpt7xvb74\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-sl9rc2-6hzgpt7xvb74/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-sl9rc2\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}}],"failure_pattern_summary":{"signature_count":52,"failure_pattern_ids":["82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54"],"matched_occurrences":52,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-21T02:00:39Z","started_at":"2026-09-21T02:00:39Z","completed_at":"2026-09-21T02:18:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T02:33:29.541494249Z","timing_metadata_checked_at":"2026-09-21T02:33:29.541494249Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T02:33:29.544751338Z","region_metadata_checked_at":"2026-09-21T02:33:29.544751338Z"},"failed_at":["e2e"],"failed_test_count":56,"occurrences":[{"row_id":"0132ff6654a022e65473c71877ab3938c8be53d34e2e469d8ad0d855e755faee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"e696af3ad9fc4fd567e40091acf3cde126175022df1890c9b16304501dcb1f10","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-wdtksw\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f5f340\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-wdtksw: failed starting cluster creation \"cp-ystream-upgrade-4-22-wdtksw\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-428nv4bnxwn5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-428nv4bnxwn5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-wdtksw\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-wdtksw\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f5f340\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-wdtksw: failed starting cluster creation \"cp-ystream-upgrade-4-22-wdtksw\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-428nv4bnxwn5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-428nv4bnxwn5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-wdtksw\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"0445f6e5de4de692ee46c1e89ea98277f616461d444474ee92d24707798501d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"ec11ee9107a30b2c86346e35f32eab5432ec779de6d222c4d10842edcb8300c4","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f96140\u003e: \n    failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-9wqv8h8p9zbx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-9wqv8h8p9zbx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f96140\u003e: \n    failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-9wqv8h8p9zbx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-9wqv8h8p9zbx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"07f4243c33a08f4ca8d4de4cc140d49c96160eee92385896539b0939a08ed311","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"e4e7d6294e99638067d43e1c2924769562a4b308f4a338b8f67ecbfb05f5bb7a","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fea2a0\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed starting cluster creation \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-t2jr6k4tmq4c\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-t2jr6k4tmq4c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fea2a0\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed starting cluster creation \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-t2jr6k4tmq4c\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-t2jr6k4tmq4c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"117444fb98a583d15da72bfe82bfa11b0e45600aa4464873c263faefa3b95f1e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"ae9e6fdf345c9d8ef9bdd095b1dd518a8384cda69ba689b6cc71189b8c70d657","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-8557jv\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ea8240\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-8557jv: failed starting cluster creation \"cp-ystream-upgrade-4-21-8557jv\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-675tr5mcqwvk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-675tr5mcqwvk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-8557jv\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-8557jv\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ea8240\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-8557jv: failed starting cluster creation \"cp-ystream-upgrade-4-21-8557jv\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-675tr5mcqwvk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-675tr5mcqwvk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-8557jv\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"1785494a61aea1d5971b05af7625ca777a162dd2237f996d7c327f8cf92b4eba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"3c2331be59160487586184247b11300e16cc85dad880af6f5f0ffff2d3dd0dbc","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-2rczz8\" with version 5.0.0-rc.1 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c983e0\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-2rczz8: failed starting cluster creation \"cluster-zstream-5-0-2rczz8\" in resourcegroup=\"rg-zstream-upgrade-5-0-95cdqxpj5hbf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-95cdqxpj5hbf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-2rczz8\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-2rczz8\" with version 5.0.0-rc.1 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c983e0\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-2rczz8: failed starting cluster creation \"cluster-zstream-5-0-2rczz8\" in resourcegroup=\"rg-zstream-upgrade-5-0-95cdqxpj5hbf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-95cdqxpj5hbf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-2rczz8\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"18eb7cee2bb4aa24cb1d1a071f18e7e4e7853052a03e0edc0db7c9823711afb4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"c2603d597733bbcf37c048127f0c54d7a0be4c7c11af9ea2b411ee0a77ae0d05","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f244a0\u003e: \n    failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-qpb4mxnx9x6p\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-qpb4mxnx9x6p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f244a0\u003e: \n    failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-qpb4mxnx9x6p\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-qpb4mxnx9x6p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"1b3bab554c8ab25117256498a696321e05e7a1a23a739c89780afab2ca91e1a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"29279c6d367e493a4c70d2b5135a4174eb8a48da083778c129968cda1ebee337","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c204e0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-ctq6xj95br5b\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-ctq6xj95br5b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c204e0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-ctq6xj95br5b\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-ctq6xj95br5b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"1eb49bba4c7db8191868aa5bc09d024562e2fae0ed55ae7317349835f2e957a8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"19fa0fb8b92f355e770eb1fe8e362e7c6df790abc78a32cf44ed59f188b7e2b0","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a14240\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-4lrv2gtf7bs7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-4lrv2gtf7bs7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a14240\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-4lrv2gtf7bs7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-4lrv2gtf7bs7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"22d7cfe16205592a3869545182831655524c864c5a07425bd0fa4d72a5364061","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"8f773d32e26d0dfbed86e9ecc627d9c5d29b2ec04038a454e10371bf42708c1c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-wrgbnp75gkw6/cluster-candidate-5-0-t8s7sr should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d02f00\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-t8s7sr: failed starting cluster creation \"cluster-candidate-5-0-t8s7sr\" in resourcegroup=\"rg-candidate-5-0-wrgbnp75gkw6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-wrgbnp75gkw6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-5-0-t8s7sr\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-wrgbnp75gkw6/cluster-candidate-5-0-t8s7sr should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d02f00\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-t8s7sr: failed starting cluster creation \"cluster-candidate-5-0-t8s7sr\" in resourcegroup=\"rg-candidate-5-0-wrgbnp75gkw6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-wrgbnp75gkw6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-5-0-t8s7sr\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"2640e25a94f19f393cb65897691ff9b962f3344ff558cd1294c28e88e634db00","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"580a97987b226e93c6d0c49e2279742bd5d28c513810045c6361dd9f03cf8074","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-qs8s65\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011f44e0\u003e: \n    failed to create HCP cluster np-noedge-qs8s65: failed starting cluster creation \"np-noedge-qs8s65\" in resourcegroup=\"rg-np-noedge-qs8s65-25h7txjtgjg9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-qs8s65-25h7txjtgjg9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-qs8s65\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-qs8s65\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011f44e0\u003e: \n    failed to create HCP cluster np-noedge-qs8s65: failed starting cluster creation \"np-noedge-qs8s65\" in resourcegroup=\"rg-np-noedge-qs8s65-25h7txjtgjg9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-qs8s65-25h7txjtgjg9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-qs8s65\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"2e39569968173ee0ab0d4dc5a9ca721c1f80bdd5b20095064d0dfd5cb0a4e0ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"16ac4a3d1553a96b79a7f5bd732b85b783fda5fb048bcd42e89d89b2626ba8d4","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000aa0630\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-rvft6vbz72q7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000aa0630\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-rvft6vbz72q7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"3121bab23088e95ced9541de2cd92244a4c0049613be654604fdf1245afeefb7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"6396f28bda025fc2bbae4ddabdd9aca6aeab3abae264e5954c84b070b3f875b5","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-q78286c7588g/cluster-candidate-4-22-h4gjpj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d02940\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-h4gjpj: failed starting cluster creation \"cluster-candidate-4-22-h4gjpj\" in resourcegroup=\"rg-candidate-4-22-q78286c7588g\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-q78286c7588g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-22-h4gjpj\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-q78286c7588g/cluster-candidate-4-22-h4gjpj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d02940\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-h4gjpj: failed starting cluster creation \"cluster-candidate-4-22-h4gjpj\" in resourcegroup=\"rg-candidate-4-22-q78286c7588g\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-q78286c7588g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-22-h4gjpj\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"3ac7bb8d5261e8d811e3c5e08c35d87f434bf4fd9fb37021bb6aaef326a331f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"36cc04b7e5713be718526ecd9e5b533015d88f8bdf4630d3c387c62b06cd507f","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001547da0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001547da0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"3efddeb3aace3d5864197b9a9211b53b548114a5edd0eefe3a2d4033f6ccfd6b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"a314fa49a11ee4e466508bbb1c5cbe99525b4cc31a134b06216dd06a4b85ad6c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-bjdhwbdj4lzm/cluster-candidate-4-20-ls9f27 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ada6a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-ls9f27: failed starting cluster creation \"cluster-candidate-4-20-ls9f27\" in resourcegroup=\"rg-candidate-4-20-bjdhwbdj4lzm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-bjdhwbdj4lzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-20-ls9f27\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-bjdhwbdj4lzm/cluster-candidate-4-20-ls9f27 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ada6a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-ls9f27: failed starting cluster creation \"cluster-candidate-4-20-ls9f27\" in resourcegroup=\"rg-candidate-4-20-bjdhwbdj4lzm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-bjdhwbdj4lzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-20-ls9f27\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"45e99a20f81d785f25a8d7155dd1aacb6367c2e8f31eb134c5d4499bc44f2ccd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"7910626bdd1f059c92afb399490324d0fa0ac7909b1455587969ce2d9e081d11","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-7h7f7p with version 4.20.38\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00174ec80\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-7h7f7p: failed starting cluster creation \"np-version-upgrade-cluster-7h7f7p\" in resourcegroup=\"rg-np-version-upgrade-7h7f7p-pdq8ldzpqzj7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-7h7f7p-pdq8ldzpqzj7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-7h7f7p\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-7h7f7p with version 4.20.38\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00174ec80\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-7h7f7p: failed starting cluster creation \"np-version-upgrade-cluster-7h7f7p\" in resourcegroup=\"rg-np-version-upgrade-7h7f7p-pdq8ldzpqzj7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-7h7f7p-pdq8ldzpqzj7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-7h7f7p\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"465f9e9fc3b3628c3d171c45dc504dcfc0a06b970f549ac85a1144c25deb081c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"8798b8c4255b3b873182eedfa6e9773524c626b9a301c47c1a4f131f313e0b26","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000996100\u003e: \n    failed to create HCP cluster cilium-cl: failed starting cluster creation \"cilium-cl\" in resourcegroup=\"cni-cilium-27whn7brwdh2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-27whn7brwdh2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cl\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000996100\u003e: \n    failed to create HCP cluster cilium-cl: failed starting cluster creation \"cilium-cl\" in resourcegroup=\"cni-cilium-27whn7brwdh2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-27whn7brwdh2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cl\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"4c9534e4335a3ed61ab81f7d2495c7a36bb3eebf4b79c2ae35affba4117cbdcb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"dc77098bca92e335b0fb329e3f3603dd68a6275b3c0782e90632a69eee1acc3a","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-x6wfmn\" with version 4.20.37 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f76600\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-x6wfmn: failed starting cluster creation \"cluster-zstream-4-20-x6wfmn\" in resourcegroup=\"rg-zstream-upgrade-4-20-fnl68dt5c6bm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-fnl68dt5c6bm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-x6wfmn\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-x6wfmn\" with version 4.20.37 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f76600\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-x6wfmn: failed starting cluster creation \"cluster-zstream-4-20-x6wfmn\" in resourcegroup=\"rg-zstream-upgrade-4-20-fnl68dt5c6bm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-fnl68dt5c6bm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-x6wfmn\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"5c01c6ae5eba9b82dcc9b5476228e7548db6d900feb1d30a3b48a19f7290b655","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"d429d81036e7016618b4101484328ed03a70c0edbf0a7b782dd4d368215f28ee","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001180100\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed starting cluster creation \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-qv5g2rk87dj4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-qv5g2rk87dj4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001180100\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed starting cluster creation \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-qv5g2rk87dj4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-qv5g2rk87dj4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"5f694e162b4a7827d41f44c52293cc28aef4447bdaf3a9b87fa3c432085ea7a0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"e5d643d0bc365ac39c86a315f14cd0bbe78cecd8f7fd681259708093265f26a9","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-f6t2k8 with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000889260\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-f6t2k8: failed starting cluster creation \"np-version-upgrade-cluster-f6t2k8\" in resourcegroup=\"rg-np-version-upgrade-f6t2k8-gqs6ng4vgqzd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-f6t2k8-gqs6ng4vgqzd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-f6t2k8\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-f6t2k8 with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000889260\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-f6t2k8: failed starting cluster creation \"np-version-upgrade-cluster-f6t2k8\" in resourcegroup=\"rg-np-version-upgrade-f6t2k8-gqs6ng4vgqzd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-f6t2k8-gqs6ng4vgqzd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-f6t2k8\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"611eb7eb515686eeeaf736c4280d1a85cbc89982afcf13c657f4576a4a5bf115","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"4af0fb2153d048ed9088447861c9bec6dc1e3ab6897c0ccd460ace40feb433b0","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004a64c0\u003e: \n    failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-2sf6rlj9tb6z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-2sf6rlj9tb6z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004a64c0\u003e: \n    failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-2sf6rlj9tb6z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-2sf6rlj9tb6z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"616c2cff7a5909f58053e5e172ab5689904a32080907185de1a6ae1042613961","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"1e10758d9271938c2300d9c8d5e6582d535e7099068afa992bb3cd1e353a9061","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f222a0\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed starting cluster creation \"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-h4nxb52wgvp8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-h4nxb52wgvp8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/missing-mi-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f222a0\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed starting cluster creation \"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-h4nxb52wgvp8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-h4nxb52wgvp8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/missing-mi-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"63822e4d7af0f693ca16690bfb7ac095871c8a49517a1fe741118f55d72ce283","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"a782ee45ffb0618ed18237cf821b9b3e921d10cf999b9078810f6aa5f36878bc","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00109a340\u003e: \n    failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-ljjxzrlv5g79\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-ljjxzrlv5g79/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00109a340\u003e: \n    failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-ljjxzrlv5g79\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-ljjxzrlv5g79/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"65a6f7bd3d769c6a4488f2596f753c99d90d3ef5b3a335175763be57c23d34c7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"3b16f4c8ac6b57b05dc789c78652f1cd0cb688d5ef7b1821b8270685f52a1d22","test_name":"Customer should be able to update node pool labels and taints","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001186140\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed starting cluster creation \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-trft9fsgtfd4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-trft9fsgtfd4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001186140\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed starting cluster creation \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-trft9fsgtfd4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-trft9fsgtfd4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6634a920911dd3d0a01ae6f1be2fc5c4e74d84e707c7b9f9d4c507986018077d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"a3332a72846a01bca504ebb4a617273366d48b4613a78a1d617fb92085c1e632","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-2g76wg\" with version 4.21.33 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d1c140\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-2g76wg: failed starting cluster creation \"cluster-zstream-4-21-2g76wg\" in resourcegroup=\"rg-zstream-upgrade-4-21-jf4dhltqzt9z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-jf4dhltqzt9z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-2g76wg\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-2g76wg\" with version 4.21.33 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d1c140\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-2g76wg: failed starting cluster creation \"cluster-zstream-4-21-2g76wg\" in resourcegroup=\"rg-zstream-upgrade-4-21-jf4dhltqzt9z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-jf4dhltqzt9z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-2g76wg\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"69c318a36bf7a5617c40b035448ebefdb0c8d7874b89dca12041f01b434f744f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"88f6a26be63494dc07c3c0a6797619890c80f87afc86fde9df4420c6cd612eb0","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a21080\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-ctcnh7zks87g\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-ctcnh7zks87g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a21080\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-ctcnh7zks87g\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-ctcnh7zks87g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6c1f4249b68d533683eeb5637887e0bc39bcbbd2191ebdb0bd4366d1f72abb23","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"ab0f29ba00a1e092f18bbe86008e731e80842d7ce025180bf57349234eb43d1f","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d843c0\u003e: \n    failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-ck5mw2gqrjlr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-ck5mw2gqrjlr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d843c0\u003e: \n    failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-ck5mw2gqrjlr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-ck5mw2gqrjlr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6f209e9bf4fd2586341d9d7a2a118596ec20f3fd50a7aae98445e1979ea77004","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"755371f92483217dc471d193d975b0e1b406bc6f6f6c8b88e00889f639022570","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010d6a20\u003e: \n    failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-wvff9rf9vjm8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-wvff9rf9vjm8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010d6a20\u003e: \n    failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-wvff9rf9vjm8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-wvff9rf9vjm8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"76886cfe4011405a4643e72734f02833460dc4f5eb5f8b783d2c0b863d4590fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"bc10fc54d7d72de0980d1057d40546af1eafd3de67a5255243d694635ff02a1b","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-77tlpl at version 4.22.14\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00142a320\u003e: \n    failed to create HCP cluster np-skip-minor-77tlpl: failed starting cluster creation \"np-skip-minor-77tlpl\" in resourcegroup=\"rg-np-skip-minor-77tlpl-8cn7ld66w96l\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-77tlpl-8cn7ld66w96l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-77tlpl\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-77tlpl at version 4.22.14\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00142a320\u003e: \n    failed to create HCP cluster np-skip-minor-77tlpl: failed starting cluster creation \"np-skip-minor-77tlpl\" in resourcegroup=\"rg-np-skip-minor-77tlpl-8cn7ld66w96l\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-77tlpl-8cn7ld66w96l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-77tlpl\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7b94ed93921b7caa0efaf830cf00dd85390f1eca6e6d6537f5f4d2b108c248a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"18210fff581546544e5d81419d0897a699e580abd777cca0f06855bee24d841c","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001498100\u003e: \n    failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-xvm84dn4wgbw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-xvm84dn4wgbw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001498100\u003e: \n    failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-xvm84dn4wgbw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-xvm84dn4wgbw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7bab1f561b807f57403db134f9a5c68e399fad838966a8f5e37e65d5aac569b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"0681b7ea641b5ced25f09bafcbbdfefd362e011a70324828804d353e457d863c","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001b9a280\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \"patch-tags-cluster\" in resourcegroup=\"patch-tags-ps4ptnfqxj84\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-ps4ptnfqxj84/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001b9a280\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \"patch-tags-cluster\" in resourcegroup=\"patch-tags-ps4ptnfqxj84\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-ps4ptnfqxj84/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7d1a790bf244c3026a11db5d15a8268f1416dc35a1388dc0863ef0e22f01d453","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"c6d131c03cb05ee28e1104fe16dc619c7100b7e358a3e982e08f4929a7438df2","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-tmlh9s\" with version 4.22.13 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c0d900\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-tmlh9s: failed starting cluster creation \"cluster-zstream-4-22-tmlh9s\" in resourcegroup=\"rg-zstream-upgrade-4-22-zfszj72t6lpb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-zfszj72t6lpb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-tmlh9s\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-tmlh9s\" with version 4.22.13 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c0d900\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-tmlh9s: failed starting cluster creation \"cluster-zstream-4-22-tmlh9s\" in resourcegroup=\"rg-zstream-upgrade-4-22-zfszj72t6lpb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-zfszj72t6lpb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-tmlh9s\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"82b8815682dff245174c8a69d84ae7c6d627d4b245ec8548cf44800fce65d17b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"455777e52442893a84fbbdf4e4b758ed5d3235191e73adf15194c0479369aa55","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b66380\u003e: \n    failed to create HCP cluster np-dg-minor-b6fkps: failed starting cluster creation \"np-dg-minor-b6fkps\" in resourcegroup=\"rg-np-dg-minor-b6fkps-5xqffsnnk5zx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-b6fkps-5xqffsnnk5zx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-b6fkps\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b66380\u003e: \n    failed to create HCP cluster np-dg-minor-b6fkps: failed starting cluster creation \"np-dg-minor-b6fkps\" in resourcegroup=\"rg-np-dg-minor-b6fkps-5xqffsnnk5zx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-b6fkps-5xqffsnnk5zx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-b6fkps\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"82d012d68c1eb5542d0c3b54b0da4239700e33cdcb4ed5882ae17b63e3503e89","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"5e440d48ec9e1080c3d34e1cd4fd140fbd749858a337b035b8b4feb211f945d3","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008c41a0\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed starting cluster creation \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-mct7k2g7g5sx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-mct7k2g7g5sx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cx-rg-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008c41a0\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed starting cluster creation \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-mct7k2g7g5sx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-mct7k2g7g5sx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cx-rg-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"8e640c05157ae4541db8cf3c61ed7550b2d5de10d9ffa5515a0b191dacb0e69e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"f3bdba49da02b82a2ec829e468c5e7e9b0c30892150ddb990e611e69a2948881","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f24360\u003e: \n    failed starting cluster creation \"private-kv-cluster\" in resourcegroup=\"private-keyvault-jmpxtp9q8nj9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-jmpxtp9q8nj9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f24360\u003e: \n    failed starting cluster creation \"private-kv-cluster\" in resourcegroup=\"private-keyvault-jmpxtp9q8nj9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-jmpxtp9q8nj9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"9a46bce9b646621dd2e8b4b1b92fa3551e0d25adde314eaa2fa8386be48f10aa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"0da1ec9baadff40a18055fd3f7e0e986a34a7c5f43a6538c19356ec1c7d570e5","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c5a120\u003e: \n    failed to create HCP cluster np-downgrade-lrxqrn: failed starting cluster creation \"np-downgrade-lrxqrn\" in resourcegroup=\"rg-np-downgrade-lrxqrn-hc7njtnnpstl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-lrxqrn-hc7njtnnpstl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-lrxqrn\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c5a120\u003e: \n    failed to create HCP cluster np-downgrade-lrxqrn: failed starting cluster creation \"np-downgrade-lrxqrn\" in resourcegroup=\"rg-np-downgrade-lrxqrn-hc7njtnnpstl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-lrxqrn-hc7njtnnpstl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-lrxqrn\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"9c9133262230f8225b802383a2db4ba8d4aa7452ee9864288259adf9ebb312aa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"e6ae1bf4b365f285ca262fed040cd94cde356a1850aa285c1035a235bcf45426","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000508800\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-mfwfn9k88txb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-mfwfn9k88txb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000508800\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-mfwfn9k88txb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-mfwfn9k88txb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"9e1e220ec0cb9f5d0fe1b4bd374d198036f0957dedd8037d7fbeec89f7a2feb5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"05b566ab0b6fad55c9b3bff90b77704a73b6b444c09b461ca4c5cee00f01dc86","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00109a2a0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-7d9kcjg4sv5f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7d9kcjg4sv5f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00109a2a0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-7d9kcjg4sv5f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7d9kcjg4sv5f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ae56ad7570e31f4769b474a8a2b6c4da56571a7bd5ead2895689e44205eef030","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"d25f4f468ca231dd940ec1f76720b7336f16c77e0512315d5b093ea7b343cb20","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00087e380\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed starting cluster creation \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-vh6gf9pbw22b\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-vh6gf9pbw22b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-delete-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00087e380\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed starting cluster creation \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-vh6gf9pbw22b\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-vh6gf9pbw22b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-delete-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"af32eacb353f9fea6a4bada4e1ed91feacda6510d69927aaac4744be0ef521c3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"590a3f0237f2092a9eca276d637fbf1b4a669fcdfd3a0f795893d3c2172b480b","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-47dnq2 with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ca2180\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-47dnq2: failed starting cluster creation \"np-version-upgrade-cluster-47dnq2\" in resourcegroup=\"rg-np-version-upgrade-47dnq2-9vdmn59k2fhf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-47dnq2-9vdmn59k2fhf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-47dnq2\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-47dnq2 with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ca2180\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-47dnq2: failed starting cluster creation \"np-version-upgrade-cluster-47dnq2\" in resourcegroup=\"rg-np-version-upgrade-47dnq2-9vdmn59k2fhf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-47dnq2-9vdmn59k2fhf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-47dnq2\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"b10c0c6416b7a6a87dc83b7c7ca2fd2c189e32783aee6ab08c0638df54b4b8ed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"ffaecd780c700ed35b8036c42ba5d4d7d78b52ebbb1d6cb607a80ed80ed11699","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bc4160\u003e: \n    failed to create HCP cluster node-limit-cluster: failed starting cluster creation \"node-limit-cluster\" in resourcegroup=\"node-limit-test-96ntf9fmk6cf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-96ntf9fmk6cf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bc4160\u003e: \n    failed to create HCP cluster node-limit-cluster: failed starting cluster creation \"node-limit-cluster\" in resourcegroup=\"node-limit-test-96ntf9fmk6cf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-96ntf9fmk6cf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"b18bac0e1622829ed7e1651ab67615a5a4af63ebf575a129df6bfafd798c38d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"7f8e0bd81fcb9cf99b92022e1b7311eae51b6fa5bfec2089cdf404029ddd088d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-dxjnpl64vzwv/cluster-candidate-4-21-vtftzj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010062c0\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-vtftzj: failed starting cluster creation \"cluster-candidate-4-21-vtftzj\" in resourcegroup=\"rg-candidate-4-21-dxjnpl64vzwv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-dxjnpl64vzwv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-21-vtftzj\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-dxjnpl64vzwv/cluster-candidate-4-21-vtftzj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010062c0\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-vtftzj: failed starting cluster creation \"cluster-candidate-4-21-vtftzj\" in resourcegroup=\"rg-candidate-4-21-dxjnpl64vzwv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-dxjnpl64vzwv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-21-vtftzj\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"b472bd07dfa5c2e758848046afda8b8589627e42faef9e55888648f3c5d1abda","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"50a755c0a2e8464dd55ccff2b738d398143063fa9fcb6c8e9d9a303b62686c8e","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-4bzr4j\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001902260\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-4bzr4j: failed starting cluster creation \"cp-ystream-upgrade-5-0-4bzr4j\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-v97jwkzbh25g\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-v97jwkzbh25g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-4bzr4j\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-4bzr4j\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001902260\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-4bzr4j: failed starting cluster creation \"cp-ystream-upgrade-5-0-4bzr4j\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-v97jwkzbh25g\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-v97jwkzbh25g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-4bzr4j\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"bd0d85c02f04eeaacd06c091b21801e77221254640d737f74060a11103a54d49","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"1879189cbcba74fc983450e35a1bfab11308c77236af8eed1e6bd3b6244c22b2","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000901320\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-qrqhb87zdk25\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-qrqhb87zdk25/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000901320\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-qrqhb87zdk25\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-qrqhb87zdk25/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"bd25cc01eebdf1b93316f846451b07e06997224152c3b2cfbc690aa581ed1e3b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"28a6981b3a0f868bdc29da712404e0cbbf87a85b52cf605ebe8896d11153d47a","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:105]: failed to begin creating HCP cluster \"admin-cred-lifecycle-c4jtmh\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007a6b20\u003e: \n    failed starting cluster creation \"admin-cred-lifecycle-c4jtmh\" in resourcegroup=\"admin-credential-lifecycle-test-jkj422c9xnf7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-jkj422c9xnf7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-c4jtmh\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:105]: failed to begin creating HCP cluster \"admin-cred-lifecycle-c4jtmh\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007a6b20\u003e: \n    failed starting cluster creation \"admin-cred-lifecycle-c4jtmh\" in resourcegroup=\"admin-credential-lifecycle-test-jkj422c9xnf7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-jkj422c9xnf7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-c4jtmh\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c59238bfef84b03638d4a89f26d888ba3df24b68875ea45803bb43a7adcdc26b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"bbd3ff3b57cac1b5c7defa7aea962b0b119e27ece94b9f5616f329662166292c","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007480e0\u003e: \n    failed starting cluster creation \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-8hcds6tdr27j\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-8hcds6tdr27j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007480e0\u003e: \n    failed starting cluster creation \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-8hcds6tdr27j\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-8hcds6tdr27j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c893ec4ad1abb14ce04e1350e4b2758761209ef0baf40ebb012140598c0b6504","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"7dc4fe6f187b9a0626115e21b0795958d4cb1e374b98db971201453cd6649636","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004e4820\u003e: \n    failed to create HCP cluster ea-cluster: failed starting cluster creation \"ea-cluster\" in resourcegroup=\"external-auth-cluster-n95clxb7lb7c\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-n95clxb7lb7c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004e4820\u003e: \n    failed to create HCP cluster ea-cluster: failed starting cluster creation \"ea-cluster\" in resourcegroup=\"external-auth-cluster-n95clxb7lb7c\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-n95clxb7lb7c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"cb110d33fd25d119f69fc66e47588ee25d9c4f68c149198435af29cf141e50cb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"b9e8311b97b87dd92f57e3830ec6d655e723c1cc70ad28ce74c7299c7a171a27","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d85020\u003e: \n    failed starting cluster creation \"agg-cluster\" in resourcegroup=\"feature-aggregation-qq5qf8jwbw7r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-qq5qf8jwbw7r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d85020\u003e: \n    failed starting cluster creation \"agg-cluster\" in resourcegroup=\"feature-aggregation-qq5qf8jwbw7r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-qq5qf8jwbw7r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"cf138340dd310edb9438d2d61bd1f45ae1f95bf825a52a81554deca0dc98d6f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"db986c61cff5f35d602da7c17b8b7b7a016c1d2c5b4b30cc917ac615b5527571","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f4e0c0\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-kfjtdtnsdjtj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-kfjtdtnsdjtj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f4e0c0\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-kfjtdtnsdjtj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-kfjtdtnsdjtj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d0f76ce468e854348a81edef9afeea4dfa58b4ee8af28b14bd7ed6706f99f02d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"011785d6925ad31e3be1446bbaccc9e75219a475623f190d1c769ef063ecffc1","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a746c0\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed starting cluster creation \"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-2vjthmq6nlmm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-2vjthmq6nlmm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a746c0\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed starting cluster creation \"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-2vjthmq6nlmm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-2vjthmq6nlmm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d2a533d7a6c5874b5703b44855e651e7fe6b0900d5acfffba985a81d06f8e28d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"2650ee4f802610a0b9db61b5b02d0fda8846ea0de6813e6abdd4fcb596ba547f","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001da6280\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-98sbnjbst5jv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-98sbnjbst5jv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001da6280\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-98sbnjbst5jv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-98sbnjbst5jv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"df66acec891b4eb943d9cfdd22ebcb5d0f69805365e865638c5d3c0a72459124","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"f25f97db92c9e15722c1a85428dc78e722463ec5ef01c365b8194c8233863d29","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f184a0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-kmsdxsgccqf9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-kmsdxsgccqf9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f184a0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-kmsdxsgccqf9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-kmsdxsgccqf9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"e1a3c4f2c91004d5c1549188d43ae07bd60deca4e994b72b7c8762a9e500e37f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"5e89885715dcefad39fb11787dca85a4d87399c80650145979fdf5c42ad1c9e7","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007fe200\u003e: \n    failed starting cluster creation \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-8bvxdqpgvvqp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-8bvxdqpgvvqp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007fe200\u003e: \n    failed starting cluster creation \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-8bvxdqpgvvqp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-8bvxdqpgvvqp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f79de44614697196b2739c9ddf56c8ff76ea0e49fab1a1b8c2d4160dc9730078","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"e066bb310cd70a3555c6c0fef06ebc6a77a2f525afac8870c31f77bf0ffe2712","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ba2440\u003e: \n    failed to create HCP cluster ea-listrrfznh: failed starting cluster creation \"ea-listrrfznh\" in resourcegroup=\"ea-list-qj9dwn57lzvc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-qj9dwn57lzvc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listrrfznh\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ba2440\u003e: \n    failed to create HCP cluster ea-listrrfznh: failed starting cluster creation \"ea-listrrfznh\" in resourcegroup=\"ea-list-qj9dwn57lzvc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-qj9dwn57lzvc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listrrfznh\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f968ad174928ef63ffa7f4151939a0187a3eb0d50c015523e3d56510940d6866","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"1bc92f7b9feb08ade6c9bfa04f41825ef4e18dd49e68329a57fcf8bf5cb87e9e","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009c8140\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gq6t85fxql89\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-gq6t85fxql89/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009c8140\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gq6t85fxql89\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-gq6t85fxql89/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"fa60d12882b914edfbfb3baba70d2ca550422a26a13afce795343d757a504b26","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"5f018a78d9584ccfe4e11889e22f11f0b0290f2b287be893271cf2172dd62aa9","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001132380\u003e: \n    failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-b7snk66zhjvq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-b7snk66zhjvq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001132380\u003e: \n    failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-b7snk66zhjvq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-b7snk66zhjvq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"fc053f593490d3573a058faa03f1b3db2cac5b2a47f8aabfe73c18ec9fc3ee6f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101854085849288704","occurred_at":"2026-09-21T02:00:39Z","failed_at":"e2e","signature_id":"2774100d49466c77dbdf0a2531d4820ebd2f58c604d43ac2fcdfcfbff4d95f38","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cce660\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-pzhxk8rzlp5m\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-pzhxk8rzlp5m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cce660\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-pzhxk8rzlp5m\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-pzhxk8rzlp5m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}}],"failure_pattern_summary":{"signature_count":56,"failure_pattern_ids":["82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54"],"matched_occurrences":56,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}