{"meta":{"date":"2026-09-22","timezone":"UTC","generated_at":"2026-10-14T21:08:03Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":118,"failed_runs":102,"runs_with_occurrences":102,"runs_with_failure_pattern_matches":99,"runs_with_unmatched_occurrences":3,"runs_without_occurrence_rows":16,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102540867792277504","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7053,"pr_state":"open","pr_sha":"3546f073a56c727f2ef4cc510ceee8663c7709da","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-22T23:29:40Z","started_at":"2026-09-22T23:29:40Z","completed_at":"2026-09-23T01:29:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T01:41:08.373642875Z","timing_metadata_checked_at":"2026-09-23T01:41:08.373642875Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T01:41:08.376331367Z","region_metadata_checked_at":"2026-09-23T01:41:08.376331367Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534129747759104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6821,"pr_state":"open","pr_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T23:02:54Z","started_at":"2026-09-22T23:02:54Z","completed_at":"2026-09-23T01:12:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T01:20:03.092715859Z","timing_metadata_checked_at":"2026-09-23T01:20:03.092715859Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T01:20:03.095628249Z","region_metadata_checked_at":"2026-09-23T01:20:03.095628249Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"e34ed39ee2e62870d394500c10cf4140c9ba6f434826a24eb603e07f16b63879","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534129747759104","occurred_at":"2026-09-22T23:02:54Z","failed_at":"alert","signature_id":"90465e28739c8b20aca20c879dc28759613fff25e486dc294036702bbd11ac8c","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-23T00:53:07Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j7759104-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-pn2sd\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-pn2sd (namespace arobit) on cluster ci01-j7759104-svc has averaged 127.3% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534076756922368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6922,"pr_state":"open","pr_sha":"87e40a07142a1457ed3f83374cc5d085c87d8f29","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T23:02:41Z","started_at":"2026-09-22T23:02:41Z","completed_at":"2026-09-23T01:12:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T01:20:03.210512668Z","timing_metadata_checked_at":"2026-09-23T01:20:03.210512668Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T01:20:03.213907057Z","region_metadata_checked_at":"2026-09-23T01:20:03.213907057Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"9d79fc958c51c2318284dcbc933d11e5d1758bf39f1e5cd5f6f5cbf93be117fb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534076756922368","occurred_at":"2026-09-22T23:02:41Z","failed_at":"alert","signature_id":"3ed4d9b039f61b590fffd2d094c8a21584d0052cbfd2d7647bd6c94de416a2ca","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-23T00:42:32Z\n Ended: 2026-09-23T00:53:32Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j6922368-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters exceeded 1 second over the past 30 minutes (cluster ci01-j6922368-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534072453566464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6820,"pr_state":"open","pr_sha":"c99d95a578f89e3e4c3bfc476a031bb4b59056f4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T23:02:40Z","started_at":"2026-09-22T23:02:40Z","completed_at":"2026-09-23T00:23:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T00:38:02.442496565Z","timing_metadata_checked_at":"2026-09-23T00:38:02.442496565Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T00:38:02.446112752Z","region_metadata_checked_at":"2026-09-23T00:38:02.446112752Z"},"failed_at":["alert","provision"],"failed_test_count":4,"occurrences":[{"row_id":"18757792190584629d629959e5fd4bea7190601cafb6234e604a5741ab973c6e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534072453566464","occurred_at":"2026-09-22T23:02:40Z","failed_at":"alert","signature_id":"2dd3e964221f66c841fca4fdbecbd25f55231ce99032e40e07a49730aa63f949","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-23T00:04:14Z\n Severity: Sev3\n Labels: alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j3566464-svc\", component=\"monitoring-infrastructure\", region=\"centralus\", severity=\"warning\"\n Description: The Arobit forwarder metrics endpoint on cluster ci01-j3566464-svc has been unreachable for 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired"}},{"row_id":"4a007d201e0f462be4db6caeeef14abc382afc90150856ce9e82b169d0b4695d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534072453566464","occurred_at":"2026-09-22T23:02:40Z","failed_at":"alert","signature_id":"a04e75c99557ace11f8b88485cfe566562bca31c71278e8a33d003f9d8dde804","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-23T00:03:29Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j3566464-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.65:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3566464/providers/microsoft.monitor/accounts/services-j3566464\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-849cb69576-lcpcf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-23T00:03:29Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j3566464-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.65:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3566464/providers/microsoft.monitor/accounts/services-j3566464\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-849cb69576-lcpcf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired"}},{"row_id":"57847fd0a3ce2a30dcccfa871bf1f0cfc3985ac217e6013f93417d7b4002cbb7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534072453566464","occurred_at":"2026-09-22T23:02:40Z","failed_at":"alert","signature_id":"e5c46939c2f0bd4411d5eb2f2df37dbc82ead1a90ce003059981d14a1a7c466f","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T23:53:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3566464-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-5x4b6\", region=\"centralus\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-5x4b6 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-22T23:53:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3566464-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-56wq6\", region=\"centralus\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-56wq6 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-22T23:53:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3566464-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-njbgt\", region=\"centralus\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-njbgt has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-22T23:53:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3566464-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-dkjc6\", region=\"centralus\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-dkjc6 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"7fcb55bd45ebbd66ba36590da10b1ac58603d3f65be6e6fd4905e8342d3267ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534072453566464","occurred_at":"2026-09-22T23:02:40Z","failed_at":"provision","signature_id":"1f7aa719c9f65ad08c7354a49de1693f07f66c1401899d7f84a949b5b6228c2b","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","test_suite":"templatize-pipeline","failure_text":"time=2026-09-22T23:47:19.977Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-22T23:47:20.376Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j3566464-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:centralus role:hcp-underlay-ci01-j3566464-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5eca8f713c64383bc545dd55f4af4020fd5db0c392d891cdc441765a2f7e52c3 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:3739bfae-90c4-46a6-9e9a-788bc6da855b region:centralus secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#3739bfae-90c4-46a6-9e9a-788bc6da855b INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-22T23:47:20.376Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T23:47:20.465Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-22T23:47:20.652Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-22T23:47:20.652Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T23:47:20.723Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T23:57:23.239Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-22T23:57:23.240Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-22T23:58:23.243Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-22T23:58:23.878Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j3566464-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:centralus role:hcp-underlay-ci01-j3566464-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5eca8f713c64383bc545dd55f4af4020fd5db0c392d891cdc441765a2f7e52c3 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:3739bfae-90c4-46a6-9e9a-788bc6da855b region:centralus secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#3739bfae-90c4-46a6-9e9a-788bc6da855b INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-22T23:58:23.878Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T23:58:24.112Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-22T23:58:24.439Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T23:58:24.439Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T23:58:24.639Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-22T23:58:25.408Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T23:58:25.408Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T23:58:25.669Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T23:58:26.134Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-22T23:58:26.496Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-22T23:58:26.679Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T23:58:26.994Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T23:58:27.236Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T23:58:27.460Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-22T23:58:27.460Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T23:58:27.560Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-23T00:08:29.375Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-23T00:08:29.375Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-23T00:09:29.384Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-23T00:09:29.909Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j3566464-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:centralus role:hcp-underlay-ci01-j3566464-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5eca8f713c64383bc545dd55f4af4020fd5db0c392d891cdc441765a2f7e52c3 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:3739bfae-90c4-46a6-9e9a-788bc6da855b region:centralus secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#3739bfae-90c4-46a6-9e9a-788bc6da855b INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-23T00:09:29.909Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T00:09:30.097Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-23T00:09:30.328Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T00:09:30.328Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T00:09:30.580Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-23T00:09:31.341Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T00:09:31.341Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T00:09:31.535Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T00:09:31.879Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-23T00:09:32.169Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-23T00:09:32.570Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T00:09:32.841Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T00:09:33.156Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T00:09:33.376Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-23T00:09:33.376Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T00:09:33.452Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-23T00:17:19.905Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"context deadline exceeded\"\ntime=2026-09-23T00:17:19.906Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","failure_pattern":"failed to roll out Helm release: context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7064/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534053163962368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7064,"pr_state":"closed","pr_sha":"f79a2b49b041574b466c35db9b025bacfc78ce0d","final_merged_sha":"f79a2b49b041574b466c35db9b025bacfc78ce0d","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T23:02:36Z","started_at":"2026-09-22T23:02:36Z","completed_at":"2026-09-23T00:55:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T00:59:15.252168669Z","timing_metadata_checked_at":"2026-09-23T00:59:15.252168669Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T00:59:15.262381534Z","region_metadata_checked_at":"2026-09-23T00:59:15.262381534Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"198580c2f16922a01d0f47b03bab7e728c0f62c8741e3790a6b6b5472a1d57da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7064/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534053163962368","occurred_at":"2026-09-22T23:02:36Z","failed_at":"alert","signature_id":"833f40b32f2367c7a516e338150ce8ef38e4e99136c3b8d58fb87da8e3aac23d","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s) (1 unknown, 1 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-23T00:19:53Z\n Ended: 2026-09-23T00:25:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3962368-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t0ca0g6h5bft9u4f3cil433bkiopk5i-t0j1r2j4w5n7o8z\", pod=\"router-77497b854-mxvbx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t0ca0g6h5bft9u4f3cil433bkiopk5i-t0j1r2j4w5n7o8z/router-77497b854-mxvbx has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"8580705ea0196366808a55f20253854f157c00aaca796cddf42d220dccb14011","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7064/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534053163962368","occurred_at":"2026-09-22T23:02:36Z","failed_at":"alert","signature_id":"dc0ddfc8aae9b431fe2054ee903a37d2a737e09a536f135d1539226ec722ed8f","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-23T00:08:48Z\n Ended: 2026-09-23T00:37:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3962368-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3962368/providers/microsoft.monitor/accounts/hcps-j3962368\", namespace=\"ocm-arohcpci01-2t0ca9hj1ibgehdpkuutjoor5mhimmon-s3k7g2g7e0s8u2n\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e1e0ac9c-ae62-4294-a73a-42eb6033d5d4\"\n Description: Pod ocm-arohcpci01-2t0ca9hj1ibgehdpkuutjoor5mhimmon-s3k7g2g7e0s8u2n/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-09-23T00:09:48Z\n Ended: 2026-09-23T00:37:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3962368-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3962368/providers/microsoft.monitor/accounts/hcps-j3962368\", namespace=\"ocm-arohcpci01-2t0ca9hj1ibgehdpkuutjoor5mhimmon-s3k7g2g7e0s8u2n\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e1e0ac9c-ae62-4294-a73a-42eb6033d5d4\"\n Description: Pod ocm-arohcpci01-2t0ca9hj1ibgehdpkuutjoor5mhimmon-s3k7g2g7e0s8u2n/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"948e38eccdc03c8cd51735b9f75f759036dbea19e9feb29c4df492b5dd6318c5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7064/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534053163962368","occurred_at":"2026-09-22T23:02:36Z","failed_at":"e2e","signature_id":"edd2ee36bfbf83bd2218ba79d6110db800244e83bcd1a31816119060e5229d1f","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-bg4xww with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016eec80\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-bg4xww: failed waiting for cluster=\"np-version-upgrade-cluster-bg4xww\" in resourcegroup=\"rg-np-version-upgrade-bg4xww-fxbpl7ffx59q\" to finish creating: GET https://rp.j3962368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0f4e4436-53e4-4539-ab16-39eefac91dac\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0f4e4436-53e4-4539-ab16-39eefac91dac\",\n \"name\": \"0f4e4436-53e4-4539-ab16-39eefac91dac\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T23:45:41.203501418Z\",\n \"endTime\": \"2026-09-23T00:14:41.784000482Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-bg4xww with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016eec80\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-bg4xww: failed waiting for cluster=\"np-version-upgrade-cluster-bg4xww\" in resourcegroup=\"rg-np-version-upgrade-bg4xww-fxbpl7ffx59q\" to finish creating: GET https://rp.j3962368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0f4e4436-53e4-4539-ab16-39eefac91dac\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0f4e4436-53e4-4539-ab16-39eefac91dac\",\n \"name\": \"0f4e4436-53e4-4539-ab16-39eefac91dac\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T23:45:41.203501418Z\",\n \"endTime\": \"2026-09-23T00:14:41.784000482Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b83cc157d9182c28f0f08926bd98e7cc3b1d3ebb6b3ab853055d264da2fddf67","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7064/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534053163962368","occurred_at":"2026-09-22T23:02:36Z","failed_at":"alert","signature_id":"41bc348af7b09437b01801e49a5b4d2b4e19f50e7146ef23ebcde6ae3d0472ca","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 5 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-23T00:19:55Z\n Ended: 2026-09-23T00:26:51Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3962368-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3962368/providers/microsoft.monitor/accounts/services-j3962368\", namespace=\"klusterlet-2t0ca9hj1ibgehdpkuutjoor5mhimmon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-r9h6m\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t0ca9hj1ibgehdpkuutjoor5mhimmon/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-23T00:19:55Z\n Ended: 2026-09-23T00:25:51Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3962368-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3962368/providers/microsoft.monitor/accounts/services-j3962368\", namespace=\"klusterlet-2t0ca9hj1ibgehdpkuutjoor5mhimmon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-r9h6m\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t0ca9hj1ibgehdpkuutjoor5mhimmon/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-23T00:19:55Z\n Ended: 2026-09-23T00:26:52Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3962368-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3962368/providers/microsoft.monitor/accounts/services-j3962368\", namespace=\"klusterlet-2t0ca9hj1ibgehdpkuutjoor5mhimmon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-r9h6m\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t0ca9hj1ibgehdpkuutjoor5mhimmon/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-23T00:19:55Z\n Ended: 2026-09-23T00:25:51Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3962368-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3962368/providers/microsoft.monitor/accounts/services-j3962368\", namespace=\"klusterlet-2t0ca9hj1ibgehdpkuutjoor5mhimmon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-r9h6m\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t0ca9hj1ibgehdpkuutjoor5mhimmon/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-23T00:19:56Z\n Ended: 2026-09-23T00:25:51Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3962368-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3962368/providers/microsoft.monitor/accounts/services-j3962368\", namespace=\"klusterlet-2t0ca9hj1ibgehdpkuutjoor5mhimmon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-r9h6m\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t0ca9hj1ibgehdpkuutjoor5mhimmon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}},{"row_id":"cdbd731d6e88abe15aa4e8f13a4f96cf41020c13eb2bc5adf34eb7fb2e01b77f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7064/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534053163962368","occurred_at":"2026-09-22T23:02:36Z","failed_at":"alert","signature_id":"fd2edebd87cfbeca482aaf300e4be69a630973901365509fd78112388229bda3","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-23T00:20:53Z\n Ended: 2026-09-23T00:34:52Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3962368-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3962368/providers/microsoft.monitor/accounts/hcps-j3962368\", namespace=\"ocm-arohcpci01-2t0ca9hj1ibgehdpkuutjoor5mhimmon-s3k7g2g7e0s8u2n\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-r9h6m\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t0ca9hj1ibgehdpkuutjoor5mhimmon-s3k7g2g7e0s8u2n/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-23T00:20:53Z\n Ended: 2026-09-23T00:39:52Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3962368-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3962368/providers/microsoft.monitor/accounts/hcps-j3962368\", namespace=\"ocm-arohcpci01-2t0ca9hj1ibgehdpkuutjoor5mhimmon-s3k7g2g7e0s8u2n\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-r9h6m\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t0ca9hj1ibgehdpkuutjoor5mhimmon-s3k7g2g7e0s8u2n/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"ed7a63a3702599d728dd49a888f5d7720bb3d96e519b6849587486f3fe52ace0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7064/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534053163962368","occurred_at":"2026-09-22T23:02:36Z","failed_at":"alert","signature_id":"a2c0bd06e35a7576d6d086fe8bc93742d833cb34906141aac4bb8ad9ab215f07","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-23T00:05:55Z\n Ended: 2026-09-23T00:33:57Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3962368-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3962368/providers/microsoft.monitor/accounts/hcps-j3962368\", namespace=\"ocm-arohcpci01-2t0ca9hj1ibgehdpkuutjoor5mhimmon-s3k7g2g7e0s8u2n\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-r9h6m\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t0ca9hj1ibgehdpkuutjoor5mhimmon-s3k7g2g7e0s8u2n/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-23T00:06:59Z\n Ended: 2026-09-23T00:37:57Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3962368-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3962368/providers/microsoft.monitor/accounts/hcps-j3962368\", namespace=\"ocm-arohcpci01-2t0ca9hj1ibgehdpkuutjoor5mhimmon-s3k7g2g7e0s8u2n\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-r9h6m\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t0ca9hj1ibgehdpkuutjoor5mhimmon-s3k7g2g7e0s8u2n/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7089/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102530666527723520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7089,"pr_state":"closed","pr_sha":"0a81896e19a7bd7d095267a0eb3a19c7d7fd94e9","final_merged_sha":"0a81896e19a7bd7d095267a0eb3a19c7d7fd94e9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T22:49:08Z","started_at":"2026-09-22T22:49:08Z","completed_at":"2026-09-23T00:52:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T00:59:15.383117619Z","timing_metadata_checked_at":"2026-09-23T00:59:15.383117619Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T00:59:15.386072009Z","region_metadata_checked_at":"2026-09-23T00:59:15.386072009Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"400b1c10f94ca66575b2c99662f88268939a2f92b108279c9566e1b013793a07","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7089/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102530666527723520","occurred_at":"2026-09-22T22:49:08Z","failed_at":"alert","signature_id":"e63f01b8163ebef955b0612083617e40ee244e5eac4deff580529f1300de341a","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-23T00:20:40Z\n Ended: 2026-09-23T00:32:36Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j7723520-svc\", component=\"frontend\", method=\"get\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters exceeded 1 second over the past 30 minutes (cluster ci01-j7723520-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102530027982688256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6359,"pr_state":"open","pr_sha":"ec8ebc721039ebe68bd095d89ea7760a1a74f924","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T22:46:36Z","started_at":"2026-09-22T22:46:36Z","completed_at":"2026-09-23T01:12:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T01:20:02.972928556Z","timing_metadata_checked_at":"2026-09-23T01:20:02.972928556Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T01:20:02.977537141Z","region_metadata_checked_at":"2026-09-23T01:20:02.977537141Z"},"failed_at":["alert","e2e","unknown"],"failed_test_count":5,"occurrences":[{"row_id":"35d47dc46a94e73bbf7694c68fa2e7a5cd8b708132478ffdeb297f4a6ca5640d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102530027982688256","occurred_at":"2026-09-22T22:46:36Z","failed_at":"alert","signature_id":"701a40a807847341bdf087ce02e65a9ab90dca8be0fe9f9ce4f46071198313de","test_name":"[aro-hcp-observability] [hcp] alert userJourneySwiftRouterAvailability1h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T23:53:49Z\n Ended: 2026-09-23T00:03:49Z\n Severity: Sev3\n Labels: alertname=\"userJourneySwiftRouterAvailability1h5m\", burn_rate_tier=\"fast\", cluster=\"ci01-j2688256-mgmt-1\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2688256/providers/microsoft.monitor/accounts/hcps-j2688256\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"swift-router-availability\"\n Description: SWIFT router-capacity availability is consuming its error budget at the fast burn rate.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"27efdba0688c010a9407237cceb9d29c447c7f9bd7965cf5f46a9ce24313bf72","failure_pattern":"alert [hcp] userJourneySwiftRouterAvailability1h5m fired","search_query":"alert [hcp] userJourneySwiftRouterAvailability1h5m fired"}},{"row_id":"386a9cc382143897bd2779f35897e796d6f01eff978f51e4d2d05cfb1f7d8087","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102530027982688256","occurred_at":"2026-09-22T22:46:36Z","failed_at":"e2e","signature_id":"acc491f73905825ad00c1e614ceac5874bb38c51bbce62f68329aa496898fecb","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001f04078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-xvhgrrnh7hjx\" to finish deleting: GET https://rp.j2688256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/177d426f-dd32-4799-8b9b-eb1aaa182ca9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/177d426f-dd32-4799-8b9b-eb1aaa182ca9\",\n \"name\": \"177d426f-dd32-4799-8b9b-eb1aaa182ca9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T23:54:44.577772615Z\",\n \"endTime\": \"2026-09-23T00:18:50.338452518Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001f04078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-xvhgrrnh7hjx\" to finish deleting: GET https://rp.j2688256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/177d426f-dd32-4799-8b9b-eb1aaa182ca9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/177d426f-dd32-4799-8b9b-eb1aaa182ca9\",\n \"name\": \"177d426f-dd32-4799-8b9b-eb1aaa182ca9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T23:54:44.577772615Z\",\n \"endTime\": \"2026-09-23T00:18:50.338452518Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3bc88c6c8876f1c2ea5668dd924adc7c81d6e0349b9850342e67442ed3fe914","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"63b6881dee37280b26b5af84b1acae95491bb45197b0575d588ea076d820c732","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102530027982688256","occurred_at":"2026-09-22T22:46:36Z","failed_at":"unknown","signature_id":"20a552c9ef65c7031e4db49271edb6fa7f672d4bde424eea2c61b746d13edef7","test_name":"Job run should complete before timeout","test_suite":"prowjob-junit","failure_text":"{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:178\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 15m0s timeout\",\"severity\":\"error\",\"time\":\"2026-09-23T00:52:30Z\"}","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f9989e753db1daf9717992966720fb7c20aebb56a0a3e0abe95adf30bb4255a7","failure_pattern":"Process did not finish before 15m0s timeout","search_query":"Process did not finish before 15m0s timeout"}},{"row_id":"e681eb8f454d0d397986b867359edfab914997d87337ed4347451149596af018","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102530027982688256","occurred_at":"2026-09-22T22:46:36Z","failed_at":"e2e","signature_id":"675a5eb91c3847f975a012a61453329e1aaa79d3dc276ce4e8bb78f6001a6575","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012b0090\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-mm2mhl4ksfsn\" to finish deleting: GET https://rp.j2688256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5d1c688d-941e-4b98-9435-e64aad496c66\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5d1c688d-941e-4b98-9435-e64aad496c66\",\n \"name\": \"5d1c688d-941e-4b98-9435-e64aad496c66\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T00:01:29.977367864Z\",\n \"endTime\": \"2026-09-23T00:25:30.446435885Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t0c9ahk4qlhegp42spvj33akbiomek9 still exists (deletion dispatched at 2026-09-23T00:01:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012b0090\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-mm2mhl4ksfsn\" to finish deleting: GET https://rp.j2688256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5d1c688d-941e-4b98-9435-e64aad496c66\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5d1c688d-941e-4b98-9435-e64aad496c66\",\n \"name\": \"5d1c688d-941e-4b98-9435-e64aad496c66\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T00:01:29.977367864Z\",\n \"endTime\": \"2026-09-23T00:25:30.446435885Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t0c9ahk4qlhegp42spvj33akbiomek9 still exists (deletion dispatched at 2026-09-23T00:01:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fdc6ca630c7a70f27c2fba6529fbd5c9f312051f7f61710808e8bed87ba9e888","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102530027982688256","occurred_at":"2026-09-22T22:46:36Z","failed_at":"e2e","signature_id":"f950417357ce3a8a0c953a3e489cb0646ab86a213b08de387c8ba03089892d50","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0013740c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-nzhgrm9s9j4m\" to finish deleting: GET https://rp.j2688256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/08842c66-3ce5-48d7-9b8b-2b725af2fdb6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/08842c66-3ce5-48d7-9b8b-2b725af2fdb6\",\n \"name\": \"08842c66-3ce5-48d7-9b8b-2b725af2fdb6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T00:01:27.518475459Z\",\n \"endTime\": \"2026-09-23T00:25:30.539881421Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t0c8kgo0dundgseri9pe4b9l9rg3k7o still exists (deletion dispatched at 2026-09-23T00:01:28Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0013740c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-nzhgrm9s9j4m\" to finish deleting: GET https://rp.j2688256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/08842c66-3ce5-48d7-9b8b-2b725af2fdb6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/08842c66-3ce5-48d7-9b8b-2b725af2fdb6\",\n \"name\": \"08842c66-3ce5-48d7-9b8b-2b725af2fdb6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T00:01:27.518475459Z\",\n \"endTime\": \"2026-09-23T00:25:30.539881421Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t0c8kgo0dundgseri9pe4b9l9rg3k7o still exists (deletion dispatched at 2026-09-23T00:01:28Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["27efdba0688c010a9407237cceb9d29c447c7f9bd7965cf5f46a9ce24313bf72","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","f3bc88c6c8876f1c2ea5668dd924adc7c81d6e0349b9850342e67442ed3fe914","f9989e753db1daf9717992966720fb7c20aebb56a0a3e0abe95adf30bb4255a7"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102519183383203840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7033,"pr_state":"closed","pr_sha":"ef821496d20065a3db001282a41d83827d994051","final_merged_sha":"ef821496d20065a3db001282a41d83827d994051","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-22T22:03:30Z","started_at":"2026-09-22T22:03:30Z","completed_at":"2026-09-22T23:57:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T00:07:09.292703475Z","timing_metadata_checked_at":"2026-09-23T00:07:09.292703475Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T00:07:09.295749463Z","region_metadata_checked_at":"2026-09-23T00:07:09.295749463Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6516/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102511455189340160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6516,"pr_state":"closed","pr_sha":"571559aeb25fe3e03ab5b299d1b520ff8d3206ca","final_merged_sha":"571559aeb25fe3e03ab5b299d1b520ff8d3206ca","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-22T21:32:48Z","started_at":"2026-09-22T21:32:48Z","completed_at":"2026-09-22T23:30:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T23:34:50.143409347Z","timing_metadata_checked_at":"2026-09-22T23:34:50.143409347Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T23:34:50.147617133Z","region_metadata_checked_at":"2026-09-22T23:34:50.147617133Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102506260145377280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7053,"pr_state":"open","pr_sha":"3546f073a56c727f2ef4cc510ceee8663c7709da","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T21:12:09Z","started_at":"2026-09-22T21:12:09Z","completed_at":"2026-09-22T23:11:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T23:23:51.279076926Z","timing_metadata_checked_at":"2026-09-22T23:23:51.279076926Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T23:23:51.282550914Z","region_metadata_checked_at":"2026-09-22T23:23:51.282550914Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"07e6966fe10f29d8e8a3610ef6e8f817ca14080bbb68ba92c5c3976f11f3062c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102506260145377280","occurred_at":"2026-09-22T21:12:09Z","failed_at":"e2e","signature_id":"4bed88c5a61f4c358d0feca50612bfa11a44b660f96b61d2ab55333e0cc202f0","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-gjkk89\" for cluster \"cluster-candidate-4-22-gjkk89\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011f1c80\u003e: \n failed to create NodePool np-gjkk89: failed waiting for nodepool=\"np-gjkk89\" for cluster \"cluster-candidate-4-22-gjkk89\" in resourcegroup=\"rg-candidate-4-22-pm8qfxg2znfc\" to finish creating: GET https://rp.j5377280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e0adbd-39c6-46a6-83d0-b884d2078d83\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e0adbd-39c6-46a6-83d0-b884d2078d83\",\n \"name\": \"76e0adbd-39c6-46a6-83d0-b884d2078d83\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T22:15:39.485216693Z\",\n \"endTime\": \"2026-09-22T22:34:46.302814783Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-gjkk89\" for cluster \"cluster-candidate-4-22-gjkk89\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011f1c80\u003e: \n failed to create NodePool np-gjkk89: failed waiting for nodepool=\"np-gjkk89\" for cluster \"cluster-candidate-4-22-gjkk89\" in resourcegroup=\"rg-candidate-4-22-pm8qfxg2znfc\" to finish creating: GET https://rp.j5377280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e0adbd-39c6-46a6-83d0-b884d2078d83\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e0adbd-39c6-46a6-83d0-b884d2078d83\",\n \"name\": \"76e0adbd-39c6-46a6-83d0-b884d2078d83\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T22:15:39.485216693Z\",\n \"endTime\": \"2026-09-22T22:34:46.302814783Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102505744988377088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7054,"pr_state":"open","pr_sha":"5126a8ab14bbdc5df1fac930f763a8387b971ef2","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-22T21:10:06Z","started_at":"2026-09-22T21:10:06Z","completed_at":"2026-09-22T23:24:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T23:34:51.154107312Z","timing_metadata_checked_at":"2026-09-22T23:34:51.154107312Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T23:34:51.158542698Z","region_metadata_checked_at":"2026-09-22T23:34:51.158542698Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102503575954395136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5846,"pr_state":"closed","pr_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","final_merged_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T21:01:29Z","started_at":"2026-09-22T21:01:29Z","completed_at":"2026-09-22T23:06:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T23:13:25.23914761Z","timing_metadata_checked_at":"2026-09-22T23:13:25.23914761Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T23:13:25.242374999Z","region_metadata_checked_at":"2026-09-22T23:13:25.242374999Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"f7128feb17acbd302fc19f78546fd1ca9fc19d9b146514dae15c4bd716a4815a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102503575954395136","occurred_at":"2026-09-22T21:01:29Z","failed_at":"alert","signature_id":"89f393992f21000a05fbc5b394889210c7304fa2f585389c34870f2a013602ad","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatency6h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T22:54:46Z\n Severity: Sev3\n Labels: alertname=\"userJourneyFrontendLatency6h30m\", cluster=\"ci01-j4395136-svc\", component=\"slo\", long_window=\"6h\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n Description: More than 6% of Frontend requests on cluster ci01-j4395136-svc exceeded 1s on both the 6h and 30m windows (6x burn of the 1% latency budget).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"addfa32c4c8681e954bead4d63f7ee51babad93b3eb924950aeb173e9712942b","failure_pattern":"alert [svc] userJourneyFrontendLatency6h30m fired","search_query":"alert [svc] userJourneyFrontendLatency6h30m fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["addfa32c4c8681e954bead4d63f7ee51babad93b3eb924950aeb173e9712942b"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102503516554661888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6820,"pr_state":"open","pr_sha":"c99d95a578f89e3e4c3bfc476a031bb4b59056f4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T21:01:10Z","started_at":"2026-09-22T21:01:10Z","completed_at":"2026-09-22T22:26:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T22:31:06.004779901Z","timing_metadata_checked_at":"2026-09-22T22:31:06.004779901Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T22:31:06.00796729Z","region_metadata_checked_at":"2026-09-22T22:31:06.00796729Z"},"failed_at":["alert","provision"],"failed_test_count":4,"occurrences":[{"row_id":"9ed9d701f3f815ff1cab3043b0d25dbb2e3be2a86e5200d114fe8916159c7ab7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102503516554661888","occurred_at":"2026-09-22T21:01:10Z","failed_at":"alert","signature_id":"8bbd752890c6930e820a263345bf9af1b4d7a0775a4b9b8de16c22d020962e5c","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T22:06:23Z\n Severity: Sev3\n Labels: alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j4661888-svc\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"warning\"\n Description: The Arobit forwarder metrics endpoint on cluster ci01-j4661888-svc has been unreachable for 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired"}},{"row_id":"a78643a3aa01e8b839df0d915cc0e51ff89ac0c81718a2297314db8fd0d065d2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102503516554661888","occurred_at":"2026-09-22T21:01:10Z","failed_at":"alert","signature_id":"787c181d5ab0cf85f109f8fa52f4249050537f88d2a9185e8486e591858ba600","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T21:56:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4661888-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-2hfrg\", region=\"westus3\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-2hfrg has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-22T21:56:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4661888-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-xc5hd\", region=\"westus3\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-xc5hd has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-22T21:56:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4661888-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-74xjw\", region=\"westus3\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-74xjw has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-22T21:56:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4661888-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-cnj5s\", region=\"westus3\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-cnj5s has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"b9c6e66ac09eb8a5d0b1629d6ab0e671b348101cbe36d3e10aca070783dceee4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102503516554661888","occurred_at":"2026-09-22T21:01:10Z","failed_at":"provision","signature_id":"45b35ae85a60aea966255b483a545628aa8222b68318189a84fac3b010fd6be2","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","test_suite":"templatize-pipeline","failure_text":"time=2026-09-22T21:50:03.851Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-22T21:50:04.426Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j4661888-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-ci01-j4661888-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5eca8f713c64383bc545dd55f4af4020fd5db0c392d891cdc441765a2f7e52c3 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:8df94b5e-187a-4556-af6a-9b474ecc0b97 region:westus3 secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#8df94b5e-187a-4556-af6a-9b474ecc0b97 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-22T21:50:04.426Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T21:50:04.653Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-22T21:50:04.887Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-22T21:50:04.887Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T21:50:05.009Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T22:00:08.502Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-22T22:00:08.503Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-22T22:01:08.503Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-22T22:01:09.036Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j4661888-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-ci01-j4661888-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5eca8f713c64383bc545dd55f4af4020fd5db0c392d891cdc441765a2f7e52c3 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:8df94b5e-187a-4556-af6a-9b474ecc0b97 region:westus3 secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#8df94b5e-187a-4556-af6a-9b474ecc0b97 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-22T22:01:09.036Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T22:01:09.387Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-22T22:01:09.975Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T22:01:09.975Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T22:01:10.243Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-22T22:01:11.277Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T22:01:11.277Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T22:01:11.584Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T22:01:12.230Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-22T22:01:13.220Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-22T22:01:13.572Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T22:01:14.205Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T22:01:14.723Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T22:01:15.282Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-22T22:01:15.282Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T22:01:15.520Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-22T22:11:18.108Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-22T22:11:18.108Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-22T22:12:18.116Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-22T22:12:19.145Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j4661888-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-ci01-j4661888-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5eca8f713c64383bc545dd55f4af4020fd5db0c392d891cdc441765a2f7e52c3 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:8df94b5e-187a-4556-af6a-9b474ecc0b97 region:westus3 secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#8df94b5e-187a-4556-af6a-9b474ecc0b97 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-22T22:12:19.145Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T22:12:19.438Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-22T22:12:19.970Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T22:12:19.970Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T22:12:20.187Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-22T22:12:21.247Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T22:12:21.247Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T22:12:21.585Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T22:12:22.170Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-22T22:12:22.930Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-22T22:12:23.284Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T22:12:23.764Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T22:12:24.305Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T22:12:24.890Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-22T22:12:24.890Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T22:12:25.135Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-22T22:20:03.973Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"context deadline exceeded\"\ntime=2026-09-22T22:20:03.973Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","failure_pattern":"failed to roll out Helm release: context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded"}},{"row_id":"e4ab1a9f067eee3f9dc19bb28705b156c7861bf4ed170e7a0cc623d1be9ed905","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102503516554661888","occurred_at":"2026-09-22T21:01:10Z","failed_at":"alert","signature_id":"624f8822ae56f0da81c94d2930d1e465c523cceb3e61be6edc365aee8089e257","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T22:06:23Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j4661888-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4661888/providers/microsoft.monitor/accounts/services-j4661888\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-849cb69576-lrr2t\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-22T22:06:23Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j4661888-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4661888/providers/microsoft.monitor/accounts/services-j4661888\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-849cb69576-lrr2t\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7064/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102499706289524736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7064,"pr_state":"closed","pr_sha":"f79a2b49b041574b466c35db9b025bacfc78ce0d","final_merged_sha":"f79a2b49b041574b466c35db9b025bacfc78ce0d","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T20:46:07Z","started_at":"2026-09-22T20:46:07Z","completed_at":"2026-09-22T22:53:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T23:02:37.121193174Z","timing_metadata_checked_at":"2026-09-22T23:02:37.121193174Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T23:02:37.124180664Z","region_metadata_checked_at":"2026-09-22T23:02:37.124180664Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"7093c4124981ef7cc37ca0860776bad475a1b777bb1a68843d013c5c44227bf1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7064/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102499706289524736","occurred_at":"2026-09-22T20:46:07Z","failed_at":"alert","signature_id":"43accc06602d22f800522fb6e09d903ef020ad90c8e254c4ba9352cfc9345f98","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatency6h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T22:28:47Z\n Severity: Sev3\n Labels: alertname=\"userJourneyFrontendLatency6h30m\", cluster=\"ci01-j9524736-svc\", component=\"slo\", long_window=\"6h\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n Description: More than 6% of Frontend requests on cluster ci01-j9524736-svc exceeded 1s on both the 6h and 30m windows (6x burn of the 1% latency budget).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"addfa32c4c8681e954bead4d63f7ee51babad93b3eb924950aeb173e9712942b","failure_pattern":"alert [svc] userJourneyFrontendLatency6h30m fired","search_query":"alert [svc] userJourneyFrontendLatency6h30m fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["addfa32c4c8681e954bead4d63f7ee51babad93b3eb924950aeb173e9712942b"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7086/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102498004782026752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7086,"pr_state":"closed","pr_sha":"cb250bc8314d562c3bc460d4aa18aa9830524b1d","final_merged_sha":"cb250bc8314d562c3bc460d4aa18aa9830524b1d","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-22T20:39:21Z","started_at":"2026-09-22T20:39:21Z","completed_at":"2026-09-22T22:38:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T22:42:03.025125922Z","timing_metadata_checked_at":"2026-09-22T22:42:03.025125922Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T22:42:03.027849512Z","region_metadata_checked_at":"2026-09-22T22:42:03.027849512Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7084/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102495173459054592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7084,"pr_state":"open","pr_sha":"0af6afb9bec01b62b2977134861c9d99023b3552","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T20:20:22Z","started_at":"2026-09-22T20:20:22Z","completed_at":"2026-09-22T22:40:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T22:52:19.459880417Z","timing_metadata_checked_at":"2026-09-22T22:52:19.459880417Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T22:52:19.462914807Z","region_metadata_checked_at":"2026-09-22T22:52:19.462914807Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"2fa95bfd952d6b22d4b5cdffb0245048d9a9d0c7ead96a0c4234847cae2fd723","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7084/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102495173459054592","occurred_at":"2026-09-22T20:20:22Z","failed_at":"alert","signature_id":"d5512e2af1827ff97e8ddfa3ae72ff959c7739aa5b0e2ca11d43d5d5a2447e6f","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T21:51:11Z\n Ended: 2026-09-22T21:57:11Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j9054592-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j9054592-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"5e61bda886e68d8339f501bdd433cbb688eb08b57beac4c70f15ca2c55329abb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7084/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102495173459054592","occurred_at":"2026-09-22T20:20:22Z","failed_at":"alert","signature_id":"214335afe92282c046d1950ce30a5f52d2bcbdf9d09adec832e75fe415c69131","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T21:42:08Z\n Ended: 2026-09-22T22:16:09Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j9054592-svc\", component=\"frontend\", method=\"get\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} exceeded 1 second over the past 30 minutes (cluster ci01-j9054592-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}},{"row_id":"a8a4ebe50e5bfe07327e1f836653a37722bbf147f2f2eed22d4c27b6888087c4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7084/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102495173459054592","occurred_at":"2026-09-22T20:20:22Z","failed_at":"e2e","signature_id":"9ff9c40b0726a271e1572a52f2a2a4d98ad6a5faa3d1535bb48b76b53cdda930","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0015ac090\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-dbblg5cjpg9w\" to finish deleting: GET https://rp.j9054592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0b7ee6c5-b14b-4d17-a809-95e9f3fb536a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0b7ee6c5-b14b-4d17-a809-95e9f3fb536a\",\n \"name\": \"0b7ee6c5-b14b-4d17-a809-95e9f3fb536a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T21:59:48.810888582Z\",\n \"endTime\": \"2026-09-22T22:23:48.20051154Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t0a3mthe182nict37k3988rs7tsgdm3 still exists (deletion dispatched at 2026-09-22T21:59:51Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0015ac090\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-dbblg5cjpg9w\" to finish deleting: GET https://rp.j9054592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0b7ee6c5-b14b-4d17-a809-95e9f3fb536a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0b7ee6c5-b14b-4d17-a809-95e9f3fb536a\",\n \"name\": \"0b7ee6c5-b14b-4d17-a809-95e9f3fb536a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T21:59:48.810888582Z\",\n \"endTime\": \"2026-09-22T22:23:48.20051154Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t0a3mthe182nict37k3988rs7tsgdm3 still exists (deletion dispatched at 2026-09-22T21:59:51Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102494708675645440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6661,"pr_state":"open","pr_sha":"7037eb839c41bcb84efcd79de023c3d5839a6a83","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T20:17:39Z","started_at":"2026-09-22T20:17:39Z","completed_at":"2026-09-22T22:31:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T22:42:02.74688317Z","timing_metadata_checked_at":"2026-09-22T22:42:02.74688317Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T22:42:02.750114959Z","region_metadata_checked_at":"2026-09-22T22:42:02.750114959Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"6fbf65185071572113618a3ec4e34f4e81e647aa9617ef7d448efca7cba66237","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102494708675645440","occurred_at":"2026-09-22T20:17:39Z","failed_at":"alert","signature_id":"c65229f531bd77886e311f6f9ef82fd40ff68bc0210a843c2ea5376a427e25e6","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T21:41:14Z\n Ended: 2026-09-22T22:20:13Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j5645440-svc\", component=\"frontend\", method=\"delete\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for delete /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j5645440-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"c1b443b4969960388b2f852f9d7d0ad50a8d4a5034a049f06288ec539cbc44f1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102494708675645440","occurred_at":"2026-09-22T20:17:39Z","failed_at":"e2e","signature_id":"8a53d3030826034995c1023916f92588c2d2bc0619bf9b3be7d8c7100951a66f","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00250e0a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-t2kdhs\" in resourcegroup=\"rg-np-version-upgrade-t2kdhs-5vn4clcwghqp\" to finish deleting: GET https://rp.j5645440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/31e9dfbb-4e6b-4928-875b-adfc122ac940\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/31e9dfbb-4e6b-4928-875b-adfc122ac940\",\n \"name\": \"31e9dfbb-4e6b-4928-875b-adfc122ac940\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T21:50:57.166246559Z\",\n \"endTime\": \"2026-09-22T22:14:57.071797622Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t0a2vdu59ka2it7qevdj75a5me0ng8e still exists (deletion dispatched at 2026-09-22T21:50:58Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00250e0a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-t2kdhs\" in resourcegroup=\"rg-np-version-upgrade-t2kdhs-5vn4clcwghqp\" to finish deleting: GET https://rp.j5645440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/31e9dfbb-4e6b-4928-875b-adfc122ac940\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/31e9dfbb-4e6b-4928-875b-adfc122ac940\",\n \"name\": \"31e9dfbb-4e6b-4928-875b-adfc122ac940\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T21:50:57.166246559Z\",\n \"endTime\": \"2026-09-22T22:14:57.071797622Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t0a2vdu59ka2it7qevdj75a5me0ng8e still exists (deletion dispatched at 2026-09-22T21:50:58Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102494423198732288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6821,"pr_state":"open","pr_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T20:00:29Z","started_at":"2026-09-22T20:00:29Z","completed_at":"2026-09-22T22:52:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T23:02:36.984920636Z","timing_metadata_checked_at":"2026-09-22T23:02:36.984920636Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T23:02:36.991069215Z","region_metadata_checked_at":"2026-09-22T23:02:36.991069215Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"463f5080b83423c08351e8ac14739c0c1603d25c3550a470af1bf75d9afe4cd5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102494423198732288","occurred_at":"2026-09-22T20:00:29Z","failed_at":"e2e","signature_id":"a08178183f0c8eb37b7968df640b81b0fe1b0c10db1e057a60161fdcf72897dc","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000f760a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-49lxzl\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-ks5lgvsbcxp8\" to finish deleting: GET https://rp.j8732288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e172962-dab6-4e45-8d0d-3db19b278332\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e172962-dab6-4e45-8d0d-3db19b278332\",\n \"name\": \"5e172962-dab6-4e45-8d0d-3db19b278332\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T22:02:28.107089842Z\",\n \"endTime\": \"2026-09-22T22:26:33.710677175Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t0ad9gsjkb0dmt6ds8umuk7neh3i2c8 still exists (deletion dispatched at 2026-09-22T22:02:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000f760a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-49lxzl\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-ks5lgvsbcxp8\" to finish deleting: GET https://rp.j8732288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e172962-dab6-4e45-8d0d-3db19b278332\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e172962-dab6-4e45-8d0d-3db19b278332\",\n \"name\": \"5e172962-dab6-4e45-8d0d-3db19b278332\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T22:02:28.107089842Z\",\n \"endTime\": \"2026-09-22T22:26:33.710677175Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t0ad9gsjkb0dmt6ds8umuk7neh3i2c8 still exists (deletion dispatched at 2026-09-22T22:02:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102490037739327488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6993,"pr_state":"open","pr_sha":"54fa83b6be2bda741a3e467491a8b8caf8a2dbe3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T19:29:34Z","started_at":"2026-09-22T19:29:34Z","completed_at":"2026-09-22T22:43:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T22:52:19.326668557Z","timing_metadata_checked_at":"2026-09-22T22:52:19.326668557Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T22:52:19.329591848Z","region_metadata_checked_at":"2026-09-22T22:52:19.329591848Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"082c5814728e5f7cb2378f63859ca5625bcb24c1ec0616667b80befffb648dbf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102490037739327488","occurred_at":"2026-09-22T19:29:34Z","failed_at":"e2e","signature_id":"3c4983ca599806389d740358987cab851287fa208512172337c9a318c08dbfa5","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00152c108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-drcfdlgkcfg5\" to finish deleting: GET https://rp.j9327488.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a61bbf5a-6f6d-430a-a670-57c9accc9081\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a61bbf5a-6f6d-430a-a670-57c9accc9081\",\n \"name\": \"a61bbf5a-6f6d-430a-a670-57c9accc9081\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T22:01:22.621456872Z\",\n \"endTime\": \"2026-09-22T22:25:28.233887055Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t0a4erurouv88b1rf2h4lagi5f4egc1 still exists (deletion dispatched at 2026-09-22T22:01:23Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00152c108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-drcfdlgkcfg5\" to finish deleting: GET https://rp.j9327488.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a61bbf5a-6f6d-430a-a670-57c9accc9081\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a61bbf5a-6f6d-430a-a670-57c9accc9081\",\n \"name\": \"a61bbf5a-6f6d-430a-a670-57c9accc9081\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T22:01:22.621456872Z\",\n \"endTime\": \"2026-09-22T22:25:28.233887055Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t0a4erurouv88b1rf2h4lagi5f4egc1 still exists (deletion dispatched at 2026-09-22T22:01:23Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0bff83ca9254e10911caedf2fb589f64c37c6fa382fe8e6504ea10e2501720e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102490037739327488","occurred_at":"2026-09-22T19:29:34Z","failed_at":"alert","signature_id":"32f305375aae0b604305ebd3d7bb6bc5f6c89d67ce915ba6ee48e5a7b097541f","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T22:23:58Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j9327488-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-9ld5r\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-9ld5r (namespace arobit) on cluster ci01-j9327488-svc has averaged 126.6% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102479316322357248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6925,"pr_state":"open","pr_sha":"9ef5982dac7a7b22811f15404810c31d0372579b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T19:15:19Z","started_at":"2026-09-22T19:15:19Z","completed_at":"2026-09-22T21:23:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T21:28:10.156597392Z","timing_metadata_checked_at":"2026-09-22T21:28:10.156597392Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T21:28:10.160489178Z","region_metadata_checked_at":"2026-09-22T21:28:10.160489178Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"8507106ba402416a1edeeb1257137708c681b93d6e57258213c596cbf92c365d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102479316322357248","occurred_at":"2026-09-22T19:15:19Z","failed_at":"e2e","signature_id":"ec2c0c0328932eb53b4b57656bf9a51b204217985c768bbe396defc59424a479","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001c52240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-cn8tj9\" in resourcegroup=\"rg-np-version-upgrade-cn8tj9-htwhhv8x7nxj\" to finish deleting: GET https://rp.j2357248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/65d5b5af-f827-49a2-b835-b18ea96d28af\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/65d5b5af-f827-49a2-b835-b18ea96d28af\",\n \"name\": \"65d5b5af-f827-49a2-b835-b18ea96d28af\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T20:42:52.522338422Z\",\n \"endTime\": \"2026-09-22T21:06:55.676641569Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t093k6i18qur3n9jj565dmnt62jj5aq still exists (deletion dispatched at 2026-09-22T20:42:54Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001c52240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-cn8tj9\" in resourcegroup=\"rg-np-version-upgrade-cn8tj9-htwhhv8x7nxj\" to finish deleting: GET https://rp.j2357248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/65d5b5af-f827-49a2-b835-b18ea96d28af\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/65d5b5af-f827-49a2-b835-b18ea96d28af\",\n \"name\": \"65d5b5af-f827-49a2-b835-b18ea96d28af\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T20:42:52.522338422Z\",\n \"endTime\": \"2026-09-22T21:06:55.676641569Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t093k6i18qur3n9jj565dmnt62jj5aq still exists (deletion dispatched at 2026-09-22T20:42:54Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7067/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102491197841870848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7067,"pr_state":"closed","pr_sha":"8ae5d545296859679e6c957bce4ca9e5af6e376c","final_merged_sha":"8ae5d545296859679e6c957bce4ca9e5af6e376c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-22T19:02:04Z","started_at":"2026-09-22T19:02:04Z","completed_at":"2026-09-22T22:32:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T22:42:02.912536605Z","timing_metadata_checked_at":"2026-09-22T22:42:02.912536605Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T22:42:02.915420996Z","region_metadata_checked_at":"2026-09-22T22:42:02.915420996Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102471925983875072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6820,"pr_state":"open","pr_sha":"c99d95a578f89e3e4c3bfc476a031bb4b59056f4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T18:22:13Z","started_at":"2026-09-22T18:22:13Z","completed_at":"2026-09-22T20:24:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T20:35:00.940048627Z","timing_metadata_checked_at":"2026-09-22T20:35:00.940048627Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T20:35:00.943182717Z","region_metadata_checked_at":"2026-09-22T20:35:00.943182717Z"},"failed_at":["alert","provision"],"failed_test_count":4,"occurrences":[{"row_id":"0e28ba6e7792e4084476d9074880139341439ccbac227772ba51331ad83aedc3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102471925983875072","occurred_at":"2026-09-22T18:22:13Z","failed_at":"alert","signature_id":"fffa35f4057212200674fb96ca760632e8bcb5f7f0ce78e035f83a8a299a9866","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T20:05:10Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j3875072-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.70:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3875072/providers/microsoft.monitor/accounts/services-j3875072\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-849cb69576-lbmz5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-22T20:05:10Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j3875072-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.70:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3875072/providers/microsoft.monitor/accounts/services-j3875072\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-849cb69576-lbmz5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired"}},{"row_id":"7984ad9bc912c89c8dda5b78b08e7674e47c690916752e0951c477c1e0377420","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102471925983875072","occurred_at":"2026-09-22T18:22:13Z","failed_at":"provision","signature_id":"a4004bc2a8c2a270ffcbd9c390ca9dbe509655978183e489d387c818f6992f6e","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","test_suite":"templatize-pipeline","failure_text":"time=2026-09-22T19:48:43.126Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-22T19:48:43.619Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j3875072-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:centralus role:hcp-underlay-ci01-j3875072-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5eca8f713c64383bc545dd55f4af4020fd5db0c392d891cdc441765a2f7e52c3 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:f4808719-7718-4dd5-a60e-1d3b526f1087 region:centralus secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#f4808719-7718-4dd5-a60e-1d3b526f1087 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-22T19:48:43.619Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T19:48:43.778Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-22T19:48:43.960Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-22T19:48:43.960Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T19:48:44.045Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T19:58:46.492Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-22T19:58:46.492Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-22T19:59:46.495Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-22T19:59:47.378Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j3875072-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:centralus role:hcp-underlay-ci01-j3875072-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5eca8f713c64383bc545dd55f4af4020fd5db0c392d891cdc441765a2f7e52c3 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:f4808719-7718-4dd5-a60e-1d3b526f1087 region:centralus secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#f4808719-7718-4dd5-a60e-1d3b526f1087 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-22T19:59:47.379Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T19:59:47.630Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-22T19:59:48.103Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T19:59:48.103Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T19:59:48.332Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-22T19:59:49.351Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T19:59:49.352Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T19:59:49.594Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T19:59:49.976Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-22T19:59:50.450Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-22T19:59:50.618Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T19:59:50.774Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T19:59:51.169Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T19:59:51.518Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-22T19:59:51.518Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T19:59:51.622Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-22T20:09:53.731Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-22T20:09:53.731Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-22T20:10:53.735Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-22T20:10:54.409Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j3875072-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:centralus role:hcp-underlay-ci01-j3875072-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5eca8f713c64383bc545dd55f4af4020fd5db0c392d891cdc441765a2f7e52c3 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:f4808719-7718-4dd5-a60e-1d3b526f1087 region:centralus secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#f4808719-7718-4dd5-a60e-1d3b526f1087 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-22T20:10:54.409Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T20:10:54.723Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-22T20:10:55.205Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T20:10:55.205Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T20:10:55.345Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-22T20:10:56.191Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T20:10:56.191Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T20:10:56.565Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T20:10:56.839Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-22T20:10:57.169Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-22T20:10:57.365Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T20:10:57.525Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T20:10:57.721Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T20:10:58.033Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-22T20:10:58.033Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T20:10:58.139Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-22T20:18:43.221Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"context deadline exceeded\"\ntime=2026-09-22T20:18:43.222Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","failure_pattern":"failed to roll out Helm release: context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded"}},{"row_id":"977f0d1886cc1ee6e79134aa2dd24cf0dc4ff9c3c7346581810a90da95be7403","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102471925983875072","occurred_at":"2026-09-22T18:22:13Z","failed_at":"alert","signature_id":"5fd4ab4b504fac9b9bbcbb453f7cec49945ab22d2c27be696660fbe03e2d4f86","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T20:05:27Z\n Severity: Sev3\n Labels: alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j3875072-svc\", component=\"monitoring-infrastructure\", region=\"centralus\", severity=\"warning\"\n Description: The Arobit forwarder metrics endpoint on cluster ci01-j3875072-svc has been unreachable for 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired"}},{"row_id":"af64d3fa0f6d2e71f24e4ba767b89090efa84e1f5fb73c04bc5e386f426338cb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102471925983875072","occurred_at":"2026-09-22T18:22:13Z","failed_at":"alert","signature_id":"9e6b361c954476898c5522ff330dd04d38e832d28242edb73b886c538ec0826c","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 5 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T19:55:09Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3875072-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-8sx8x\", region=\"centralus\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-8sx8x has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-22T19:55:09Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3875072-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-zpdt2\", region=\"centralus\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-zpdt2 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-22T19:55:09Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3875072-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-pc755\", region=\"centralus\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-pc755 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-22T19:55:09Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3875072-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-kzj4q\", region=\"centralus\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-kzj4q has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-09-22T19:57:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3875072-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-749558849-6khd4\", region=\"centralus\", severity=\"warning\"\n Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-749558849-6khd4 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102462125476679680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6752,"pr_state":"closed","pr_sha":"d433c21668cfdc66f98262f9099c8086b4397248","final_merged_sha":"d433c21668cfdc66f98262f9099c8086b4397248","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-22T18:08:24Z","started_at":"2026-09-22T18:08:24Z","completed_at":"2026-09-22T20:20:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T20:24:20.674248106Z","timing_metadata_checked_at":"2026-09-22T20:24:20.674248106Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T20:24:20.679574877Z","region_metadata_checked_at":"2026-09-22T20:24:20.679574877Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102463947360702464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5846,"pr_state":"closed","pr_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","final_merged_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T18:00:43Z","started_at":"2026-09-22T18:00:43Z","completed_at":"2026-09-22T20:42:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T20:45:34.539113188Z","timing_metadata_checked_at":"2026-09-22T20:45:34.539113188Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T20:45:34.542883575Z","region_metadata_checked_at":"2026-09-22T20:45:34.542883575Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"0fa7cc88f5e2d871ba950ec86b28af0a573ac48b7bca79e7d4bfc496bdeb4c5e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102463947360702464","occurred_at":"2026-09-22T18:00:43Z","failed_at":"e2e","signature_id":"14e32cd9da960e8003053642ebd37f344e6211daa26684d90da6de88a269dbbe","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00084a288\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-68rjtkpsqm7b\" to finish deleting: GET https://rp.j0702464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6084bb0c-7df8-4842-ab2e-b53a4d94db1d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6084bb0c-7df8-4842-ab2e-b53a4d94db1d\",\n \"name\": \"6084bb0c-7df8-4842-ab2e-b53a4d94db1d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T19:42:12.055385484Z\",\n \"endTime\": \"2026-09-22T20:06:15.982381047Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t089ppdctio8jds461l0pe41vnps034 still exists (deletion dispatched at 2026-09-22T19:42:13Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00084a288\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-68rjtkpsqm7b\" to finish deleting: GET https://rp.j0702464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6084bb0c-7df8-4842-ab2e-b53a4d94db1d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6084bb0c-7df8-4842-ab2e-b53a4d94db1d\",\n \"name\": \"6084bb0c-7df8-4842-ab2e-b53a4d94db1d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T19:42:12.055385484Z\",\n \"endTime\": \"2026-09-22T20:06:15.982381047Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t089ppdctio8jds461l0pe41vnps034 still exists (deletion dispatched at 2026-09-22T19:42:13Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"21a3b5e7fe4e4fc2e9978a633063a279ead4d0d137a0798e1b5bf21a9f54142c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102463947360702464","occurred_at":"2026-09-22T18:00:43Z","failed_at":"alert","signature_id":"251163f0fa415413d3fc13e342aa1d4c5b3c3761b29974ac0d10e823d63b76d3","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T20:26:03Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j0702464-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-7j8m6\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-7j8m6 (namespace arobit) on cluster ci01-j0702464-svc has averaged 125.4% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"7cdf6e27607c26c4a0c53b98443bcfd961be0047db10350c91ae7890d9bb6c6e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102463947360702464","occurred_at":"2026-09-22T18:00:43Z","failed_at":"e2e","signature_id":"d868d40ddc424d3fed5c1fcfc795706e5f9bff04fe8acad2628a131a0fc96977","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc002455530\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc002455530\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}},{"row_id":"c803f05ed5de905f7343cee780ebd1b1013a4c0967642ea6f9c506fce6aa9c55","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102463947360702464","occurred_at":"2026-09-22T18:00:43Z","failed_at":"e2e","signature_id":"909fdafce84c913a896925a9962c4ba8be3902b29c7ec52c177584072fc1104f","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0017a8228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-jq6hhtmkbzt4\" to finish deleting: GET https://rp.j0702464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff92afaf-c9d4-443d-b74a-95c2f3e15d15\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff92afaf-c9d4-443d-b74a-95c2f3e15d15\",\n \"name\": \"ff92afaf-c9d4-443d-b74a-95c2f3e15d15\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T20:01:15.219491093Z\",\n \"endTime\": \"2026-09-22T20:25:19.472856279Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t089usm9gvf5gahfe2druepcei6i824 still exists (deletion dispatched at 2026-09-22T20:01:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0017a8228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-jq6hhtmkbzt4\" to finish deleting: GET https://rp.j0702464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff92afaf-c9d4-443d-b74a-95c2f3e15d15\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff92afaf-c9d4-443d-b74a-95c2f3e15d15\",\n \"name\": \"ff92afaf-c9d4-443d-b74a-95c2f3e15d15\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T20:01:15.219491093Z\",\n \"endTime\": \"2026-09-22T20:25:19.472856279Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t089usm9gvf5gahfe2druepcei6i824 still exists (deletion dispatched at 2026-09-22T20:01:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102458797258706944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7020,"pr_state":"closed","pr_sha":"6cb0492059ca17ab9fe7fffef3fab0908d31e190","final_merged_sha":"6cb0492059ca17ab9fe7fffef3fab0908d31e190","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T17:48:09Z","started_at":"2026-09-22T17:48:09Z","completed_at":"2026-09-22T20:25:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T20:35:01.23607422Z","timing_metadata_checked_at":"2026-09-22T20:35:01.23607422Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T20:35:01.238778411Z","region_metadata_checked_at":"2026-09-22T20:35:01.238778411Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"11ba98185f66d71947e039952879824f0e76fa7ae0eaf26fa2dc69492d884a33","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102458797258706944","occurred_at":"2026-09-22T17:48:09Z","failed_at":"alert","signature_id":"ab1804d22b808f7b28e0e2b5c1233de332584f8e97e10dfaf11d22bf3d5d0c59","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T20:12:36Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j8706944-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-5w4zq\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-5w4zq (namespace arobit) on cluster ci01-j8706944-svc has averaged 125.4% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"7bb0c27bb5d8298f7fa819b71043146640cb03da9926d197639f9e813dfa8ab9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102458797258706944","occurred_at":"2026-09-22T17:48:09Z","failed_at":"alert","signature_id":"21b04b293f9281378dcbec21bbaa132777f4a608518f67fcb04c950d001b5d21","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T19:32:14Z\n Ended: 2026-09-22T19:38:14Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j8706944-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j8706944-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"b3a6c5d5edaa3c12962e28115cfd5cde8fb2c1702e9abb9c11f052bf4acca0f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102458797258706944","occurred_at":"2026-09-22T17:48:09Z","failed_at":"e2e","signature_id":"354c4acb28abc19875c6408355327707c18ba1fb5780d90c1a40b6aa98eb5e4c","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011083d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-zzfkr9\" in resourcegroup=\"rg-np-skip-minor-zzfkr9-gb95nn9vpwbx\" to finish deleting: GET https://rp.j8706944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8986523a-9554-40d0-9d12-c9d8f53ca3b2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8986523a-9554-40d0-9d12-c9d8f53ca3b2\",\n \"name\": \"8986523a-9554-40d0-9d12-c9d8f53ca3b2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T19:40:48.963450447Z\",\n \"endTime\": \"2026-09-22T20:04:49.182718803Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t082ht679tj6q9hdfrhosuc2f437sef still exists (deletion dispatched at 2026-09-22T19:40:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011083d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-zzfkr9\" in resourcegroup=\"rg-np-skip-minor-zzfkr9-gb95nn9vpwbx\" to finish deleting: GET https://rp.j8706944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8986523a-9554-40d0-9d12-c9d8f53ca3b2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8986523a-9554-40d0-9d12-c9d8f53ca3b2\",\n \"name\": \"8986523a-9554-40d0-9d12-c9d8f53ca3b2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T19:40:48.963450447Z\",\n \"endTime\": \"2026-09-22T20:04:49.182718803Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t082ht679tj6q9hdfrhosuc2f437sef still exists (deletion dispatched at 2026-09-22T19:40:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"dfa59339ac0d7bcb16e19480aa8ead239395398b196635377bca06c52618210c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102458797258706944","occurred_at":"2026-09-22T17:48:09Z","failed_at":"e2e","signature_id":"2c1a5e84b6f1cde3b61bd78d82756fb1fc0ea6de3271eaa74fb9e6ccf95e2230","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011de180\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-wgqfmcc7mttj\" to finish deleting: GET https://rp.j8706944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a537a987-ddb8-49ee-9e92-79fbe69e65a6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a537a987-ddb8-49ee-9e92-79fbe69e65a6\",\n \"name\": \"a537a987-ddb8-49ee-9e92-79fbe69e65a6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T19:44:00.645194734Z\",\n \"endTime\": \"2026-09-22T20:08:09.172667585Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t082ge25c3lks1mu3j5dtlgjr539r8q still exists (deletion dispatched at 2026-09-22T19:44:04Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011de180\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-wgqfmcc7mttj\" to finish deleting: GET https://rp.j8706944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a537a987-ddb8-49ee-9e92-79fbe69e65a6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a537a987-ddb8-49ee-9e92-79fbe69e65a6\",\n \"name\": \"a537a987-ddb8-49ee-9e92-79fbe69e65a6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T19:44:00.645194734Z\",\n \"endTime\": \"2026-09-22T20:08:09.172667585Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t082ge25c3lks1mu3j5dtlgjr539r8q still exists (deletion dispatched at 2026-09-22T19:44:04Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102456083485298688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6752,"pr_state":"closed","pr_sha":"d433c21668cfdc66f98262f9099c8086b4397248","final_merged_sha":"d433c21668cfdc66f98262f9099c8086b4397248","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T17:22:16Z","started_at":"2026-09-22T17:22:16Z","completed_at":"2026-09-22T18:01:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T18:17:43.059917626Z","timing_metadata_checked_at":"2026-09-22T18:17:43.059917626Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-22T18:17:43.063788614Z","region_metadata_checked_at":"2026-09-22T19:20:45.242012344Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"7e56608a05da529f5d6aea63f16d78f6bd1f9336b7570a35aa7f6f9245255a9e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102456083485298688","occurred_at":"2026-09-22T17:22:16Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102448371858935808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4878,"pr_state":"closed","pr_sha":"244d0c16bcabe8284755cf089f7bb166cd38a435","final_merged_sha":"244d0c16bcabe8284755cf089f7bb166cd38a435","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T17:15:09Z","started_at":"2026-09-22T17:15:09Z","completed_at":"2026-09-22T19:39:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T19:52:14.396562Z","timing_metadata_checked_at":"2026-09-22T19:52:14.396562Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T19:52:14.400135387Z","region_metadata_checked_at":"2026-09-22T19:52:14.400135387Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"07962be29cfc8dbf30a908175a60f3c362915548f4615e070b690d1b5d29d6a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102448371858935808","occurred_at":"2026-09-22T17:15:09Z","failed_at":"alert","signature_id":"c84a9da375a2e42b5ebe685d998166743cab67da7aaa85511f0c2a9396b4c3d3","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T18:50:27Z\n Ended: 2026-09-22T19:06:27Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j8935808-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters has exceeded 250ms over the past 30 minutes (cluster ci01-j8935808-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"43ef098c8b9d4ce5afe3a68a9eb32e8fc3333a209d4ee2653b5474407a61cab4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102448371858935808","occurred_at":"2026-09-22T17:15:09Z","failed_at":"e2e","signature_id":"eebf124ff7cd8e6579cd38d8f2821cfc541502c99089f74d0f5e0f453236134a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-zck558\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001112760\u003e: \n DNS for route host agnhost-e2e-sample-app-wrj85.apps.aro.h6n7n8r8p3m8n7w.1qn2.j8935808.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-wrj85.apps.aro.h6n7n8r8p3m8n7w.1qn2.j8935808.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wrj85.apps.aro.h6n7n8r8p3m8n7w.1qn2.j8935808.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-zck558\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001112760\u003e: \n DNS for route host agnhost-e2e-sample-app-wrj85.apps.aro.h6n7n8r8p3m8n7w.1qn2.j8935808.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-wrj85.apps.aro.h6n7n8r8p3m8n7w.1qn2.j8935808.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wrj85.apps.aro.h6n7n8r8p3m8n7w.1qn2.j8935808.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-wrj85.apps.aro.h6n7n8r8p3m8n7w.1qn2.j8935808.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-wrj85.apps.aro.h6n7n8r8p3m8n7w.1qn2.j8935808.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wrj85.apps.aro.h6n7n8r8p3m8n7w.1qn2.j8935808.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded"}},{"row_id":"771e07e26b4e04112a4b4154d2d5fd311c252c28cb8e770e8d7f0253dc6e8b93","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102448371858935808","occurred_at":"2026-09-22T17:15:09Z","failed_at":"e2e","signature_id":"99198eea28d9b68ecc39071e0c0c9a858751bd46d9f111c48b130fa5cb7ea428","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00107a1e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-sfwkw6t6m866\" to finish deleting: GET https://rp.j8935808.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2c2bbf72-d927-4c9c-b696-4f1f281075fe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2c2bbf72-d927-4c9c-b696-4f1f281075fe\",\n \"name\": \"2c2bbf72-d927-4c9c-b696-4f1f281075fe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T18:57:36.550691584Z\",\n \"endTime\": \"2026-09-22T19:21:39.730536375Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t07dfina4snjdhqqt1e8v0e6qcnbvh6 still exists (deletion dispatched at 2026-09-22T18:57:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00107a1e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-sfwkw6t6m866\" to finish deleting: GET https://rp.j8935808.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2c2bbf72-d927-4c9c-b696-4f1f281075fe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2c2bbf72-d927-4c9c-b696-4f1f281075fe\",\n \"name\": \"2c2bbf72-d927-4c9c-b696-4f1f281075fe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T18:57:36.550691584Z\",\n \"endTime\": \"2026-09-22T19:21:39.730536375Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t07dfina4snjdhqqt1e8v0e6qcnbvh6 still exists (deletion dispatched at 2026-09-22T18:57:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a669db305e7e989aa514d4664b1b8565649e671dcc546d4b0ac05e5c9cdee3cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102448371858935808","occurred_at":"2026-09-22T17:15:09Z","failed_at":"alert","signature_id":"74c245df6880b6c37600b0c6bbfe6a2573fd5ffd089e497ae36eca834db88eb6","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T19:25:37Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j8935808-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-mfjw9\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-mfjw9 (namespace arobit) on cluster ci01-j8935808-svc has averaged 127.2% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102443148557946880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6821,"pr_state":"open","pr_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T17:01:22Z","started_at":"2026-09-22T17:01:22Z","completed_at":"2026-09-22T19:02:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T19:20:45.363418752Z","timing_metadata_checked_at":"2026-09-22T19:20:45.363418752Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T19:20:45.366703341Z","region_metadata_checked_at":"2026-09-22T19:20:45.366703341Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"4eae0e37fa545599ee87acf4a9e7cba040e2d379b1e1d731dbac07fe2796c0e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102443148557946880","occurred_at":"2026-09-22T17:01:22Z","failed_at":"alert","signature_id":"543e7b8c3c4a28ef5a756b6c10eb0692ee1db036dee1df85b90761212f5b3fe4","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T17:42:06Z\n Ended: 2026-09-22T18:17:10Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7946880-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7946880-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"5d4dbb0ebb5735dce053e65bebea5ffa80b4f3c54c5a4d020555b22fc6d0e3be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102443148557946880","occurred_at":"2026-09-22T17:01:22Z","failed_at":"e2e","signature_id":"aa1c09867367e81459be0baa88cdeec15ab5c644b3ef04cb4a9c498a3a36962c","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00124a138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-rztn7tgf9xgj\" to finish deleting: GET https://rp.j7946880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2caaa78a-cc90-4669-8d96-1a0204de9f98\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2caaa78a-cc90-4669-8d96-1a0204de9f98\",\n \"name\": \"2caaa78a-cc90-4669-8d96-1a0204de9f98\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T18:21:26.160080988Z\",\n \"endTime\": \"2026-09-22T18:45:29.126992957Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t06sdvhi784tdu6s15hf9hcnp7dpebk still exists (deletion dispatched at 2026-09-22T18:21:28Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00124a138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-rztn7tgf9xgj\" to finish deleting: GET https://rp.j7946880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2caaa78a-cc90-4669-8d96-1a0204de9f98\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2caaa78a-cc90-4669-8d96-1a0204de9f98\",\n \"name\": \"2caaa78a-cc90-4669-8d96-1a0204de9f98\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T18:21:26.160080988Z\",\n \"endTime\": \"2026-09-22T18:45:29.126992957Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t06sdvhi784tdu6s15hf9hcnp7dpebk still exists (deletion dispatched at 2026-09-22T18:21:28Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9e8516a7e2254e3cdf3ce80176fab02e091922d42ba91b970d67b532a9c09bf9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102443148557946880","occurred_at":"2026-09-22T17:01:22Z","failed_at":"alert","signature_id":"fcc48db5d3cd87f0fff18e34375d136c7b3ab27aa3495f73b687e0b8a00d91a7","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T18:19:06Z\n Ended: 2026-09-22T18:28:06Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j7946880-svc\", component=\"frontend\", method=\"delete\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: More than 5% of frontend requests for delete /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j7946880-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}},{"row_id":"bea5a64af89289581d44a33796064ea77d8f60313dbffabfffac07cc8da07f17","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102443148557946880","occurred_at":"2026-09-22T17:01:22Z","failed_at":"alert","signature_id":"04b8fb1a73deff541d6dac26b2975c6beebcaf18714f64bb6b47b58d5279c5eb","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T18:47:34Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j7946880-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-674px\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-674px (namespace arobit) on cluster ci01-j7946880-svc has averaged 127.1% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7079/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102443130908315648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7079,"pr_state":"closed","pr_sha":"a66c056ace7a6a9e73944032b0924658c1a5ec03","final_merged_sha":"a66c056ace7a6a9e73944032b0924658c1a5ec03","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-22T17:01:00Z","started_at":"2026-09-22T17:01:00Z","completed_at":"2026-09-22T18:45:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T18:59:56.483436592Z","timing_metadata_checked_at":"2026-09-22T18:59:56.483436592Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T18:59:56.486489977Z","region_metadata_checked_at":"2026-09-22T18:59:56.486489977Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6858/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102436820074631168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6858,"pr_state":"open","pr_sha":"e82abf6c62e655e7382ad8fff8ebc115c4416eb8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-22T16:36:12Z","started_at":"2026-09-22T16:36:12Z","completed_at":"2026-09-22T18:40:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T18:59:56.065873765Z","timing_metadata_checked_at":"2026-09-22T18:59:56.065873765Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T18:59:56.069477848Z","region_metadata_checked_at":"2026-09-22T18:59:56.069477848Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102428363070115840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6503,"pr_state":"closed","pr_sha":"6c5cfd09b9bbf1585b1cd37220da16677f327452","final_merged_sha":"a0bf660bfdf32222374a82406e0907cae4b53869","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T16:02:36Z","started_at":"2026-09-22T16:02:36Z","completed_at":"2026-09-22T18:11:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T18:28:04.910797045Z","timing_metadata_checked_at":"2026-09-22T18:28:04.910797045Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T18:28:04.915068729Z","region_metadata_checked_at":"2026-09-22T18:28:04.915068729Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"3651be6960d2cf7477798b6d289cad25db673442fa704bccc971eb8535a03846","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102428363070115840","occurred_at":"2026-09-22T16:02:36Z","failed_at":"alert","signature_id":"495668b1f8ea5dee71d0e8f473148935322994b6ef938eef0f10e2bfcaf03ca6","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T17:30:47Z\n Ended: 2026-09-22T17:38:47Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0115840-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.199:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0115840/providers/microsoft.monitor/accounts/services-j0115840\", namespace=\"klusterlet-2t06amajdmngpo94r6274auvf1ibbeql\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-mpcb7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t06amajdmngpo94r6274auvf1ibbeql/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-22T17:30:47Z\n Ended: 2026-09-22T17:37:47Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0115840-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.199:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0115840/providers/microsoft.monitor/accounts/services-j0115840\", namespace=\"klusterlet-2t06amajdmngpo94r6274auvf1ibbeql\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-mpcb7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t06amajdmngpo94r6274auvf1ibbeql/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}},{"row_id":"3a90e29924de3fc15027eb7ee045881a5e24bf7fd37256cb3bc3ee1c3a51e654","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102428363070115840","occurred_at":"2026-09-22T16:02:36Z","failed_at":"alert","signature_id":"da3ffd0280a3f1ffcc889e6cdc2362c3329776d834e1983ae2fd4b9b85c57e98","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T17:17:06Z\n Ended: 2026-09-22T17:45:02Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0115840-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.199:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0115840/providers/microsoft.monitor/accounts/hcps-j0115840\", namespace=\"ocm-arohcpci01-2t06amajdmngpo94r6274auvf1ibbeql-x5r1r1c7h5x7z3i\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-mpcb7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t06amajdmngpo94r6274auvf1ibbeql-x5r1r1c7h5x7z3i/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-22T17:17:06Z\n Ended: 2026-09-22T17:49:03Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0115840-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.199:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0115840/providers/microsoft.monitor/accounts/hcps-j0115840\", namespace=\"ocm-arohcpci01-2t06amajdmngpo94r6274auvf1ibbeql-x5r1r1c7h5x7z3i\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-mpcb7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t06amajdmngpo94r6274auvf1ibbeql-x5r1r1c7h5x7z3i/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"646e2c4966d2bf7baffc81827e2c3c2f2c7963fe4b60a0066f1389f4e23c6e3b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102428363070115840","occurred_at":"2026-09-22T16:02:36Z","failed_at":"e2e","signature_id":"8dc69387aa9b7456b52151ccc6ab4f71148e68674b5c416f1655d1db2f47a7b6","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-gp68bk\" with version 4.20.38 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011cba0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-gp68bk: failed waiting for cluster=\"cluster-zstream-4-20-gp68bk\" in resourcegroup=\"rg-zstream-upgrade-4-20-qcsqzkhckgg6\" to finish creating: GET https://rp.j0115840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/65ff5f37-d01d-4885-92e1-a54bf276ecdf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/65ff5f37-d01d-4885-92e1-a54bf276ecdf\",\n \"name\": \"65ff5f37-d01d-4885-92e1-a54bf276ecdf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:56:56.107653498Z\",\n \"endTime\": \"2026-09-22T17:26:04.415591039Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-gp68bk\" with version 4.20.38 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011cba0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-gp68bk: failed waiting for cluster=\"cluster-zstream-4-20-gp68bk\" in resourcegroup=\"rg-zstream-upgrade-4-20-qcsqzkhckgg6\" to finish creating: GET https://rp.j0115840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/65ff5f37-d01d-4885-92e1-a54bf276ecdf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/65ff5f37-d01d-4885-92e1-a54bf276ecdf\",\n \"name\": \"65ff5f37-d01d-4885-92e1-a54bf276ecdf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:56:56.107653498Z\",\n \"endTime\": \"2026-09-22T17:26:04.415591039Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"718e4d3a6fc55ca6c6a8893e5459fee1cde02f91af5755123c3f1fa3b26bcb66","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102428363070115840","occurred_at":"2026-09-22T16:02:36Z","failed_at":"alert","signature_id":"4d9ddf325a1d033b77f42eb5f81efe7623906a23240e70802d8b883ed1fe0a88","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T17:19:59Z\n Ended: 2026-09-22T17:44:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0115840-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.199:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0115840/providers/microsoft.monitor/accounts/hcps-j0115840\", namespace=\"ocm-arohcpci01-2t06amajdmngpo94r6274auvf1ibbeql-x5r1r1c7h5x7z3i\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"603dfbeb-4dca-4193-8523-f1040f307d47\"\n Description: Pod ocm-arohcpci01-2t06amajdmngpo94r6274auvf1ibbeql-x5r1r1c7h5x7z3i/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-09-22T17:19:59Z\n Ended: 2026-09-22T17:48:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0115840-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.199:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0115840/providers/microsoft.monitor/accounts/hcps-j0115840\", namespace=\"ocm-arohcpci01-2t06amajdmngpo94r6274auvf1ibbeql-x5r1r1c7h5x7z3i\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"603dfbeb-4dca-4193-8523-f1040f307d47\"\n Description: Pod ocm-arohcpci01-2t06amajdmngpo94r6274auvf1ibbeql-x5r1r1c7h5x7z3i/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"ee4df1dabbc6f6ca219f5029976353bf7dc82f6546b62fe1cdd3c07dbc56381e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102428363070115840","occurred_at":"2026-09-22T16:02:36Z","failed_at":"e2e","signature_id":"e3177ca20277054a46f71328a257ac164787c1802403c4af7bcab8b46b4dbf8d","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.007s.\nat least one schedule should have a LastBackupTime\nExpected\n \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.007s.\nat least one schedule should have a LastBackupTime\nExpected\n \u003cbool\u003e: ...\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7067/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102428355495202816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7067,"pr_state":"closed","pr_sha":"8ae5d545296859679e6c957bce4ca9e5af6e376c","final_merged_sha":"8ae5d545296859679e6c957bce4ca9e5af6e376c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T16:02:20Z","started_at":"2026-09-22T16:02:20Z","completed_at":"2026-09-22T18:07:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T18:17:43.61577232Z","timing_metadata_checked_at":"2026-09-22T18:17:43.61577232Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T18:17:43.619070009Z","region_metadata_checked_at":"2026-09-22T18:17:43.619070009Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3e1f97ed1b1753883377cd1a55f13309bf4da94bd39d5ca48a8fd7b69a2188f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7067/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102428355495202816","occurred_at":"2026-09-22T16:02:20Z","failed_at":"e2e","signature_id":"99cd53e640ecfd1e9759c1c134bdd8a22ad634239849ea0a227a303aa002a83c","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011c40f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-m9nlwn\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-cwk2zbwbl8df\" to finish deleting: GET https://rp.j5202816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e162fce1-69ff-47e2-800c-9e9a7da178b9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e162fce1-69ff-47e2-800c-9e9a7da178b9\",\n \"name\": \"e162fce1-69ff-47e2-800c-9e9a7da178b9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T17:20:42.567454265Z\",\n \"endTime\": \"2026-09-22T17:44:47.318967939Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t0673hl1rs5ia5vgvj3arfsusevukrr still exists (deletion dispatched at 2026-09-22T17:20:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011c40f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-m9nlwn\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-cwk2zbwbl8df\" to finish deleting: GET https://rp.j5202816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e162fce1-69ff-47e2-800c-9e9a7da178b9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e162fce1-69ff-47e2-800c-9e9a7da178b9\",\n \"name\": \"e162fce1-69ff-47e2-800c-9e9a7da178b9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T17:20:42.567454265Z\",\n \"endTime\": \"2026-09-22T17:44:47.318967939Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t0673hl1rs5ia5vgvj3arfsusevukrr still exists (deletion dispatched at 2026-09-22T17:20:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102425288674643968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6661,"pr_state":"open","pr_sha":"610c52d2114e9dfe27ff4fd62f91d69882254e25","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T15:50:24Z","started_at":"2026-09-22T15:50:24Z","completed_at":"2026-09-22T18:00:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T18:17:42.909505015Z","timing_metadata_checked_at":"2026-09-22T18:17:42.909505015Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T18:17:42.913825801Z","region_metadata_checked_at":"2026-09-22T18:17:42.913825801Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"4f6e836833e700d7f3cc85c577d244190f089de4d0a54ab2a4828a64f0004e50","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102425288674643968","occurred_at":"2026-09-22T15:50:24Z","failed_at":"e2e","signature_id":"abf2bef05c9e4545e9170966e3053a83da97d8f9fa426e9e835fa70973be1bc9","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001718168\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-sr9mdkjlp5d6\" to finish deleting: GET https://rp.j4643968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/db9563c3-fa41-4db8-95de-b8fb6d868146\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/db9563c3-fa41-4db8-95de-b8fb6d868146\",\n \"name\": \"db9563c3-fa41-4db8-95de-b8fb6d868146\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T17:18:42.990998968Z\",\n \"endTime\": \"2026-09-22T17:42:42.450346964Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t05uv4e9fgg9ln37cnv09e5j15gkrtl still exists (deletion dispatched at 2026-09-22T17:18:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001718168\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-sr9mdkjlp5d6\" to finish deleting: GET https://rp.j4643968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/db9563c3-fa41-4db8-95de-b8fb6d868146\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/db9563c3-fa41-4db8-95de-b8fb6d868146\",\n \"name\": \"db9563c3-fa41-4db8-95de-b8fb6d868146\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T17:18:42.990998968Z\",\n \"endTime\": \"2026-09-22T17:42:42.450346964Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t05uv4e9fgg9ln37cnv09e5j15gkrtl still exists (deletion dispatched at 2026-09-22T17:18:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424385783599104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6900,"pr_state":"open","pr_sha":"3e95ed9ff5e45dc55bd5826f736736e54e4ba393","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T15:46:49Z","started_at":"2026-09-22T15:46:49Z","completed_at":"2026-09-22T18:05:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T18:17:43.419398358Z","timing_metadata_checked_at":"2026-09-22T18:17:43.419398358Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T18:17:43.423788444Z","region_metadata_checked_at":"2026-09-22T18:17:43.423788444Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"07c572a5ddf1370d802a70b59ef6c345cfbad0f24b81c843da0fe5bf672ce8ae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424385783599104","occurred_at":"2026-09-22T15:46:49Z","failed_at":"e2e","signature_id":"f8cd22fd3ccc506847bbe331efeae788ed395ca93a5288623e9115e942ee9261","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000f30168\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-dpkm5d989phc\" to finish deleting: GET https://rp.j3599104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5496eb36-fed0-418b-b4a4-1bbb730977c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5496eb36-fed0-418b-b4a4-1bbb730977c5\",\n \"name\": \"5496eb36-fed0-418b-b4a4-1bbb730977c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T17:23:06.139309331Z\",\n \"endTime\": \"2026-09-22T17:47:12.014996899Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t061bpco7v1l3lgu542dvncm4grjrq2 still exists (deletion dispatched at 2026-09-22T17:23:07Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000f30168\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-dpkm5d989phc\" to finish deleting: GET https://rp.j3599104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5496eb36-fed0-418b-b4a4-1bbb730977c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5496eb36-fed0-418b-b4a4-1bbb730977c5\",\n \"name\": \"5496eb36-fed0-418b-b4a4-1bbb730977c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T17:23:06.139309331Z\",\n \"endTime\": \"2026-09-22T17:47:12.014996899Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t061bpco7v1l3lgu542dvncm4grjrq2 still exists (deletion dispatched at 2026-09-22T17:23:07Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bd100514cc39f8b9345b711bcace5137846ff6400fea2ec542c5137cd39ec4c2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424385783599104","occurred_at":"2026-09-22T15:46:49Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T16:52:14Z\n Ended: 2026-09-22T16:58:11Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3599104-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"cca67eb1e88a0b319a1811e5e32e66da26fd9e14069ffb9960a28587b221ce6a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424385783599104","occurred_at":"2026-09-22T15:46:49Z","failed_at":"alert","signature_id":"47de9b40c36841857354a6bfe3c0daa443eefe7d83e1b1fd6b65637d19ada076","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T17:50:10Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j3599104-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-79p2m\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-79p2m (namespace arobit) on cluster ci01-j3599104-svc has averaged 124.6% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6889,"pr_state":"open","pr_sha":"f82618a6c52b284d099a808dae1e5682131caa59","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T15:46:19Z","started_at":"2026-09-22T15:46:19Z","completed_at":"2026-09-22T18:00:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T18:17:43.287980485Z","timing_metadata_checked_at":"2026-09-22T18:17:43.287980485Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T18:17:43.290751676Z","region_metadata_checked_at":"2026-09-22T18:17:43.290751676Z"},"failed_at":["alert","e2e"],"failed_test_count":63,"occurrences":[{"row_id":"0290d48fb96e9ccda67f360c86fb5bb2056d733106ed88bcc52990b0baa3fc83","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"db89aa3bdd096aab58b24117e783390f8fd4094ede4f720ea86d619334ecbdf5","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00088c7a0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-6t86q5nsgrh9\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/315fcf5a-f184-4c52-a71c-88102dbc1533\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/315fcf5a-f184-4c52-a71c-88102dbc1533\",\n \"name\": \"315fcf5a-f184-4c52-a71c-88102dbc1533\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:26.20084073Z\",\n \"endTime\": \"2026-09-22T17:05:34.327315045Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00088c7a0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-6t86q5nsgrh9\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/315fcf5a-f184-4c52-a71c-88102dbc1533\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/315fcf5a-f184-4c52-a71c-88102dbc1533\",\n \"name\": \"315fcf5a-f184-4c52-a71c-88102dbc1533\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:26.20084073Z\",\n \"endTime\": \"2026-09-22T17:05:34.327315045Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0616db21a0039614a60c2acc7b45a7e0469e3ae5a77fd94cb1c8739c5912f2b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"6edd720649594e35198f47a4712e7d929e2cc171394ccec97afd8ce85932120e","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f62760\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-fbdrrsb4f4qb\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/673f7571-13b3-4bc0-b005-ad87a3d3f04c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/673f7571-13b3-4bc0-b005-ad87a3d3f04c\",\n \"name\": \"673f7571-13b3-4bc0-b005-ad87a3d3f04c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:35:51.009988979Z\",\n \"endTime\": \"2026-09-22T17:04:54.001140954Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f62760\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-fbdrrsb4f4qb\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/673f7571-13b3-4bc0-b005-ad87a3d3f04c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/673f7571-13b3-4bc0-b005-ad87a3d3f04c\",\n \"name\": \"673f7571-13b3-4bc0-b005-ad87a3d3f04c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:35:51.009988979Z\",\n \"endTime\": \"2026-09-22T17:04:54.001140954Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"083814cae7a55822db75f5c520ba272e724c21d0974f5d084d73f1cb2eddf378","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"0af445052595e06d74cd6d8fdc079d5bff2df4e78361aa64d110c0c33f929996","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d06700\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-b5tsprkj7fb4\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/20e6f9e0-2e0c-4d2d-81f1-3b3a636ac461\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/20e6f9e0-2e0c-4d2d-81f1-3b3a636ac461\",\n \"name\": \"20e6f9e0-2e0c-4d2d-81f1-3b3a636ac461\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:09.86553733Z\",\n \"endTime\": \"2026-09-22T17:05:14.887023579Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d06700\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-b5tsprkj7fb4\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/20e6f9e0-2e0c-4d2d-81f1-3b3a636ac461\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/20e6f9e0-2e0c-4d2d-81f1-3b3a636ac461\",\n \"name\": \"20e6f9e0-2e0c-4d2d-81f1-3b3a636ac461\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:09.86553733Z\",\n \"endTime\": \"2026-09-22T17:05:14.887023579Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0c4148285c4a0e232bdf532edd4a110475b9971da25b77486f6bcc3526a6fd1e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"fb1ed664d9909c01299048a39e7b19384c539f31e18fd1ee83b1cea24e43310a","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:88]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004f49c0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-tq6grpmch9pl\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9ef2ab64-6f5a-4ada-b76a-08072358a3a6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9ef2ab64-6f5a-4ada-b76a-08072358a3a6\",\n \"name\": \"9ef2ab64-6f5a-4ada-b76a-08072358a3a6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:35:50.115809759Z\",\n \"endTime\": \"2026-09-22T17:04:54.124624103Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:88]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004f49c0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-tq6grpmch9pl\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9ef2ab64-6f5a-4ada-b76a-08072358a3a6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9ef2ab64-6f5a-4ada-b76a-08072358a3a6\",\n \"name\": \"9ef2ab64-6f5a-4ada-b76a-08072358a3a6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:35:50.115809759Z\",\n \"endTime\": \"2026-09-22T17:04:54.124624103Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"12e7cb6b729cbb8d3d0aa240ecd531b8f76714cf96896f62fae170287be886ed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"c26d00b8f632469299803121e5e542c1bd6d48e71cf443b1fc9c6b56a9980e1c","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00110caa0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-z8xc4gxhmf84\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/12bbf6a8-0927-462a-952a-3bd89cfcdbb5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/12bbf6a8-0927-462a-952a-3bd89cfcdbb5\",\n \"name\": \"12bbf6a8-0927-462a-952a-3bd89cfcdbb5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T17:13:22.791211135Z\",\n \"endTime\": \"2026-09-22T17:42:28.16969575Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00110caa0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-z8xc4gxhmf84\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/12bbf6a8-0927-462a-952a-3bd89cfcdbb5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/12bbf6a8-0927-462a-952a-3bd89cfcdbb5\",\n \"name\": \"12bbf6a8-0927-462a-952a-3bd89cfcdbb5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T17:13:22.791211135Z\",\n \"endTime\": \"2026-09-22T17:42:28.16969575Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"17379ac6e8f41459152daa4fc9a523cb87259f3b3a041b9508e5b17836d3d243","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"667049967e319dc03b972259bd67db46a9a1814fcd21bfddde8fd56aa73f0ea4","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00107a300\u003e: \n GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/82f81316-7913-421b-adf2-73a13e0c018f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/82f81316-7913-421b-adf2-73a13e0c018f\",\n \"name\": \"82f81316-7913-421b-adf2-73a13e0c018f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:37:13.699747469Z\",\n \"endTime\": \"2026-09-22T17:06:14.29231712Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00107a300\u003e: \n GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/82f81316-7913-421b-adf2-73a13e0c018f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/82f81316-7913-421b-adf2-73a13e0c018f\",\n \"name\": \"82f81316-7913-421b-adf2-73a13e0c018f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:37:13.699747469Z\",\n \"endTime\": \"2026-09-22T17:06:14.29231712Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"184fd24a17f951d00474ef1caf4c3380012e0e4aecf9b6c2462274dccf9da3fd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"e56e9654c5bbe6050e3fd8a85052c273961dcc81b47c66b91c206ee1411a60b3","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-mqgx5p with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001a4ea0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-mqgx5p: failed waiting for cluster=\"np-version-upgrade-cluster-mqgx5p\" in resourcegroup=\"rg-np-version-upgrade-mqgx5p-sf4vhtxlsk5p\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b85ee66-6ecf-4aec-abd2-44ca10c2169c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b85ee66-6ecf-4aec-abd2-44ca10c2169c\",\n \"name\": \"7b85ee66-6ecf-4aec-abd2-44ca10c2169c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:28.335722772Z\",\n \"endTime\": \"2026-09-22T17:05:33.994048214Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-mqgx5p with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001a4ea0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-mqgx5p: failed waiting for cluster=\"np-version-upgrade-cluster-mqgx5p\" in resourcegroup=\"rg-np-version-upgrade-mqgx5p-sf4vhtxlsk5p\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b85ee66-6ecf-4aec-abd2-44ca10c2169c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b85ee66-6ecf-4aec-abd2-44ca10c2169c\",\n \"name\": \"7b85ee66-6ecf-4aec-abd2-44ca10c2169c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:28.335722772Z\",\n \"endTime\": \"2026-09-22T17:05:33.994048214Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"199c160bd44eb3c40264b36fae6e083e543bc69ead54f0d8fd40fcca3b15eaaf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"8c36d32a0259f7da9b9f1ef40b52c366005c76cbf09bab0837045aa60989b093","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f8cb00\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-v9tlgczmbzdv\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7ebec1a-d39e-4159-8ec0-c162b81238f9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7ebec1a-d39e-4159-8ec0-c162b81238f9\",\n \"name\": \"f7ebec1a-d39e-4159-8ec0-c162b81238f9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:11.943105645Z\",\n \"endTime\": \"2026-09-22T17:05:14.704022029Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f8cb00\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-v9tlgczmbzdv\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7ebec1a-d39e-4159-8ec0-c162b81238f9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7ebec1a-d39e-4159-8ec0-c162b81238f9\",\n \"name\": \"f7ebec1a-d39e-4159-8ec0-c162b81238f9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:11.943105645Z\",\n \"endTime\": \"2026-09-22T17:05:14.704022029Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1abff4a74b75a93652ef6583f306cfe757cff8877038f96de235000987c9d3c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"1526d73081656cbe45f7dcb1e8eda1497f2bff26b20c12f7d59db1211cf00231","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e4c800\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-vh5mlhlwh5jf\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/745705aa-7ec2-488a-b45f-576a0d31985e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/745705aa-7ec2-488a-b45f-576a0d31985e\",\n \"name\": \"745705aa-7ec2-488a-b45f-576a0d31985e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:19.073544004Z\",\n \"endTime\": \"2026-09-22T17:05:24.167779386Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e4c800\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-vh5mlhlwh5jf\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/745705aa-7ec2-488a-b45f-576a0d31985e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/745705aa-7ec2-488a-b45f-576a0d31985e\",\n \"name\": \"745705aa-7ec2-488a-b45f-576a0d31985e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:19.073544004Z\",\n \"endTime\": \"2026-09-22T17:05:24.167779386Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1c5b5360759febea8d1135f56ad0202ecb5d325a54646ee1eac49a2f6d5a723f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"f790b8b598528f695078414dcffe0f7fff8c6bd324401299d0482a3254ab3f04","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011842a0\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-67zjc9d45r89\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9ef3226a-042d-43f6-9a4e-08f44f519818\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9ef3226a-042d-43f6-9a4e-08f44f519818\",\n \"name\": \"9ef3226a-042d-43f6-9a4e-08f44f519818\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:12.003244297Z\",\n \"endTime\": \"2026-09-22T17:05:15.598887881Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011842a0\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-67zjc9d45r89\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9ef3226a-042d-43f6-9a4e-08f44f519818\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9ef3226a-042d-43f6-9a4e-08f44f519818\",\n \"name\": \"9ef3226a-042d-43f6-9a4e-08f44f519818\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:12.003244297Z\",\n \"endTime\": \"2026-09-22T17:05:15.598887881Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1cb335bef33c2db98f1a9bf9e69f65c8e3be76554202b5fe719edd8e1ec2aa80","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"8319add2c226ddff82deec727b005791655329b782a15be502de9fbf36fde07b","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-ntd64c\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa0ac0\u003e: \n failed to create HCP cluster np-noedge-ntd64c: failed waiting for cluster=\"np-noedge-ntd64c\" in resourcegroup=\"rg-np-noedge-ntd64c-g6pwtg6d4hs9\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a13d95b-4e01-482c-9152-e5605fd1f2ba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a13d95b-4e01-482c-9152-e5605fd1f2ba\",\n \"name\": \"8a13d95b-4e01-482c-9152-e5605fd1f2ba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:15.242676605Z\",\n \"endTime\": \"2026-09-22T17:05:23.926389731Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-ntd64c\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa0ac0\u003e: \n failed to create HCP cluster np-noedge-ntd64c: failed waiting for cluster=\"np-noedge-ntd64c\" in resourcegroup=\"rg-np-noedge-ntd64c-g6pwtg6d4hs9\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a13d95b-4e01-482c-9152-e5605fd1f2ba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a13d95b-4e01-482c-9152-e5605fd1f2ba\",\n \"name\": \"8a13d95b-4e01-482c-9152-e5605fd1f2ba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:15.242676605Z\",\n \"endTime\": \"2026-09-22T17:05:23.926389731Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1d89b80834429a1e8c06e5660efc05054f10d099aa2e93ab96a02c233ae86701","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"b9b50758a1aa006036c7c1ee58af90cd509c7b1900722ef0229b46ed2f5e8563","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"1d970047287009c503b26f286edd73e1e9081826f26505b45662d96964ae028f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"1ec61b1664ff3f185c22e8708e3354eaa939cbb4709b979c70208bbf84d4cf25","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"42781e15bc1b77679bbd1fda7afcab888bd256804d8bc00b2e3e391e7f5d9698","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f90e20\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-rlzd6dwx88l5\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3097da51-addb-491f-9447-dc8f70dfa80a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3097da51-addb-491f-9447-dc8f70dfa80a\",\n \"name\": \"3097da51-addb-491f-9447-dc8f70dfa80a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:19.895011411Z\",\n \"endTime\": \"2026-09-22T17:05:24.149753544Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f90e20\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-rlzd6dwx88l5\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3097da51-addb-491f-9447-dc8f70dfa80a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3097da51-addb-491f-9447-dc8f70dfa80a\",\n \"name\": \"3097da51-addb-491f-9447-dc8f70dfa80a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:19.895011411Z\",\n \"endTime\": \"2026-09-22T17:05:24.149753544Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"23dea5a5fb18bb9538441e5ee7376585bc24ad74d2331d8987a6f401d7df03b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"595136f34e3717abb66fa01ecb7cbd2f7ea1e74ae44dce8c1a0aa154c62f402e","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00091bb80\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-q96f99txrjlq\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d26034f6-b329-4ae9-9bed-cb3c0606e891\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d26034f6-b329-4ae9-9bed-cb3c0606e891\",\n \"name\": \"d26034f6-b329-4ae9-9bed-cb3c0606e891\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:37:47.299219553Z\",\n \"endTime\": \"2026-09-22T17:06:53.972888116Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00091bb80\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-q96f99txrjlq\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d26034f6-b329-4ae9-9bed-cb3c0606e891\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d26034f6-b329-4ae9-9bed-cb3c0606e891\",\n \"name\": \"d26034f6-b329-4ae9-9bed-cb3c0606e891\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:37:47.299219553Z\",\n \"endTime\": \"2026-09-22T17:06:53.972888116Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2467281ff142ef1182342c6a794d0e1f8bbbbf02892a52342fefebaf2b6613bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"28a878be48996c1a5644504392baa72a6c06a93c17f7a9adb5a48a03c1f748a4","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001082820\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-8sldg8lrtk8d\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c62e5766-be7f-4bc2-a830-e28cf7dd6767\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c62e5766-be7f-4bc2-a830-e28cf7dd6767\",\n \"name\": \"c62e5766-be7f-4bc2-a830-e28cf7dd6767\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:35:50.929819703Z\",\n \"endTime\": \"2026-09-22T17:04:54.254355371Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001082820\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-8sldg8lrtk8d\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c62e5766-be7f-4bc2-a830-e28cf7dd6767\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c62e5766-be7f-4bc2-a830-e28cf7dd6767\",\n \"name\": \"c62e5766-be7f-4bc2-a830-e28cf7dd6767\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:35:50.929819703Z\",\n \"endTime\": \"2026-09-22T17:04:54.254355371Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2b49a11e92d6cf22b99dac3cf1402eca43cdba8a476880e3e7134e5a640e2883","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"b99e1feb6a62e4be8de2a2c9f91e219add3d2ebb4d288578917ab29ec2dbf2e0","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009066e0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-x8qpd8lw49vm\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ee2f8ce5-9bed-48be-aeb0-cb61a7a4a93d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ee2f8ce5-9bed-48be-aeb0-cb61a7a4a93d\",\n \"name\": \"ee2f8ce5-9bed-48be-aeb0-cb61a7a4a93d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:12.016571667Z\",\n \"endTime\": \"2026-09-22T17:05:14.805027784Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009066e0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-x8qpd8lw49vm\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ee2f8ce5-9bed-48be-aeb0-cb61a7a4a93d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ee2f8ce5-9bed-48be-aeb0-cb61a7a4a93d\",\n \"name\": \"ee2f8ce5-9bed-48be-aeb0-cb61a7a4a93d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:12.016571667Z\",\n \"endTime\": \"2026-09-22T17:05:14.805027784Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"384e702b47d26badf176ec69f6660a6bcf886ad4379caeec4d3ad9a938a7272f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"5fa71470de1f76cf4fb095baee252cc2163afa119633dbab401e106d7fb2bf18","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017446a0\u003e: \n failed to create HCP cluster np-dg-minor-8wc8tn: failed waiting for cluster=\"np-dg-minor-8wc8tn\" in resourcegroup=\"rg-np-dg-minor-8wc8tn-22g8c5n8hhdx\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc38e9a7-2ed9-408e-a32b-6383c0505eaf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc38e9a7-2ed9-408e-a32b-6383c0505eaf\",\n \"name\": \"bc38e9a7-2ed9-408e-a32b-6383c0505eaf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:17.910968507Z\",\n \"endTime\": \"2026-09-22T17:05:24.133881522Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017446a0\u003e: \n failed to create HCP cluster np-dg-minor-8wc8tn: failed waiting for cluster=\"np-dg-minor-8wc8tn\" in resourcegroup=\"rg-np-dg-minor-8wc8tn-22g8c5n8hhdx\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc38e9a7-2ed9-408e-a32b-6383c0505eaf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc38e9a7-2ed9-408e-a32b-6383c0505eaf\",\n \"name\": \"bc38e9a7-2ed9-408e-a32b-6383c0505eaf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:17.910968507Z\",\n \"endTime\": \"2026-09-22T17:05:24.133881522Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"39f3f1e3164ca4a4f2dcd59d3f902d4aa353f5179dac4ae30cfca18ec22a10b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"alert","signature_id":"870d0b24a6bee5d59390ff5f28a0045b37e45b6c73b08247aec63cfac9af5bdf","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T17:43:54Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4475392-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.3:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4475392/providers/microsoft.monitor/accounts/hcps-j4475392\", namespace=\"ocm-arohcpci01-2t06119diqdpvr443dmiciulp17qu6qb-b8x0f3j9h1k5s1m\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-pwzhh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t06119diqdpvr443dmiciulp17qu6qb-b8x0f3j9h1k5s1m/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-22T17:43:54Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4475392-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.3:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4475392/providers/microsoft.monitor/accounts/hcps-j4475392\", namespace=\"ocm-arohcpci01-2t06119diqdpvr443dmiciulp17qu6qb-b8x0f3j9h1k5s1m\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-pwzhh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t06119diqdpvr443dmiciulp17qu6qb-b8x0f3j9h1k5s1m/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-22T17:45:55Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4475392-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.3:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4475392/providers/microsoft.monitor/accounts/hcps-j4475392\", namespace=\"ocm-arohcpci01-2t06119diqdpvr443dmiciulp17qu6qb-b8x0f3j9h1k5s1m\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-pwzhh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t06119diqdpvr443dmiciulp17qu6qb-b8x0f3j9h1k5s1m/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-22T17:45:55Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4475392-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.3:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4475392/providers/microsoft.monitor/accounts/hcps-j4475392\", namespace=\"ocm-arohcpci01-2t06119diqdpvr443dmiciulp17qu6qb-b8x0f3j9h1k5s1m\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-pwzhh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t06119diqdpvr443dmiciulp17qu6qb-b8x0f3j9h1k5s1m/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"3cd27a8fed7739cecc44d1a8b4cbd7260074aef07ec59105c7bf70c41f804fff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"ab7ef6dcf373c206c62413a791dfd97f4b51a92220f453b5335be9a51f844c5f","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cdea20\u003e: \n failed to create HCP cluster np-downgrade-smt8b4: failed waiting for cluster=\"np-downgrade-smt8b4\" in resourcegroup=\"rg-np-downgrade-smt8b4-p5xmqllchnwc\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7ee63103-3f10-4b67-a05a-795798c7c6c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7ee63103-3f10-4b67-a05a-795798c7c6c5\",\n \"name\": \"7ee63103-3f10-4b67-a05a-795798c7c6c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:26.373656246Z\",\n \"endTime\": \"2026-09-22T17:05:34.304731328Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cdea20\u003e: \n failed to create HCP cluster np-downgrade-smt8b4: failed waiting for cluster=\"np-downgrade-smt8b4\" in resourcegroup=\"rg-np-downgrade-smt8b4-p5xmqllchnwc\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7ee63103-3f10-4b67-a05a-795798c7c6c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7ee63103-3f10-4b67-a05a-795798c7c6c5\",\n \"name\": \"7ee63103-3f10-4b67-a05a-795798c7c6c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:26.373656246Z\",\n \"endTime\": \"2026-09-22T17:05:34.304731328Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"41e3e37f2b32d6fea4ebbdd7c50dc6729544e393aef34a4faa92fe5e08c4b5fb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"4f2dccbb23c4d3203abba3ef48c29b8b62112b27094207d9ce163be8ca02b232","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fdc6e0\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-zc4swxsfpjsc\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bfeaa5f3-bbb7-4a53-bbaf-a03f3b564a40\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bfeaa5f3-bbb7-4a53-bbaf-a03f3b564a40\",\n \"name\": \"bfeaa5f3-bbb7-4a53-bbaf-a03f3b564a40\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:27.276605223Z\",\n \"endTime\": \"2026-09-22T17:05:34.330785345Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fdc6e0\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-zc4swxsfpjsc\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bfeaa5f3-bbb7-4a53-bbaf-a03f3b564a40\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bfeaa5f3-bbb7-4a53-bbaf-a03f3b564a40\",\n \"name\": \"bfeaa5f3-bbb7-4a53-bbaf-a03f3b564a40\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:27.276605223Z\",\n \"endTime\": \"2026-09-22T17:05:34.330785345Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"41f70e21be1a15cd97f307f46fddbf82b469cbed0bb183857fcb497b202bfce0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"27b5b5c1e8a00a07e0d9a24204eaa220c56b06307d69b9cae9ceede561c51215","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d2ec20\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-kx88zsldj79h\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f0670065-55c5-420c-ad0f-2da7b18f6e3d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f0670065-55c5-420c-ad0f-2da7b18f6e3d\",\n \"name\": \"f0670065-55c5-420c-ad0f-2da7b18f6e3d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:26.610305452Z\",\n \"endTime\": \"2026-09-22T17:05:34.360693634Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d2ec20\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-kx88zsldj79h\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f0670065-55c5-420c-ad0f-2da7b18f6e3d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f0670065-55c5-420c-ad0f-2da7b18f6e3d\",\n \"name\": \"f0670065-55c5-420c-ad0f-2da7b18f6e3d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:26.610305452Z\",\n \"endTime\": \"2026-09-22T17:05:34.360693634Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"509f871e07038579d15b54bb8bcaa8375479ad5a3de57b8ef9dde11611ea0fc7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"77d324f886ea8df5e4ca54d8f6e4c7809e792140fd13f531964f9a2e4ad353ae","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-jjgb97zqjz2j/cluster-candidate-4-21-gpl8gp should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ab88e0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-gpl8gp: failed waiting for cluster=\"cluster-candidate-4-21-gpl8gp\" in resourcegroup=\"rg-candidate-4-21-jjgb97zqjz2j\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31cfd032-17f0-4e3f-8638-2dd7570fdc49\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31cfd032-17f0-4e3f-8638-2dd7570fdc49\",\n \"name\": \"31cfd032-17f0-4e3f-8638-2dd7570fdc49\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:22.453860288Z\",\n \"endTime\": \"2026-09-22T17:05:24.148111981Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-jjgb97zqjz2j/cluster-candidate-4-21-gpl8gp should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ab88e0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-gpl8gp: failed waiting for cluster=\"cluster-candidate-4-21-gpl8gp\" in resourcegroup=\"rg-candidate-4-21-jjgb97zqjz2j\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31cfd032-17f0-4e3f-8638-2dd7570fdc49\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31cfd032-17f0-4e3f-8638-2dd7570fdc49\",\n \"name\": \"31cfd032-17f0-4e3f-8638-2dd7570fdc49\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:22.453860288Z\",\n \"endTime\": \"2026-09-22T17:05:24.148111981Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5397af520937c5004b98a73e5ea9887ad799ebc9c37ac93ca828da58769fbbc9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"879e3fe90aa85b2e72d47865b800cf4566cf8b2cd76a787b032c5fd1932bee1d","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aea5a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-nptb84x7txrv\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9dca1d2-d589-496e-bb76-2944a2a36071\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9dca1d2-d589-496e-bb76-2944a2a36071\",\n \"name\": \"a9dca1d2-d589-496e-bb76-2944a2a36071\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:37:41.11913075Z\",\n \"endTime\": \"2026-09-22T17:06:44.122367318Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aea5a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-nptb84x7txrv\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9dca1d2-d589-496e-bb76-2944a2a36071\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9dca1d2-d589-496e-bb76-2944a2a36071\",\n \"name\": \"a9dca1d2-d589-496e-bb76-2944a2a36071\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:37:41.11913075Z\",\n \"endTime\": \"2026-09-22T17:06:44.122367318Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"54d9958955d2b5486f02a56cc4d72b23ca04ef7170cebaf49c67726cf4e26ff3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"b8806b22f97f5a8d736ae5b95ccf82001ed6f6ba3eeaac0cbfbc48297d8b069d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-gcds9cqvk6jx/cluster-candidate-4-20-lbcq4j should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fd81e0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-lbcq4j: failed waiting for cluster=\"cluster-candidate-4-20-lbcq4j\" in resourcegroup=\"rg-candidate-4-20-gcds9cqvk6jx\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/58f294f8-155f-4a52-8225-6c82a85d1c93\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/58f294f8-155f-4a52-8225-6c82a85d1c93\",\n \"name\": \"58f294f8-155f-4a52-8225-6c82a85d1c93\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:10.883840418Z\",\n \"endTime\": \"2026-09-22T17:05:14.699323611Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-gcds9cqvk6jx/cluster-candidate-4-20-lbcq4j should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fd81e0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-lbcq4j: failed waiting for cluster=\"cluster-candidate-4-20-lbcq4j\" in resourcegroup=\"rg-candidate-4-20-gcds9cqvk6jx\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/58f294f8-155f-4a52-8225-6c82a85d1c93\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/58f294f8-155f-4a52-8225-6c82a85d1c93\",\n \"name\": \"58f294f8-155f-4a52-8225-6c82a85d1c93\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:10.883840418Z\",\n \"endTime\": \"2026-09-22T17:05:14.699323611Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5a615168777d26ad7d8ee46dd74f903158ae4423e8eadcca7c0c4a64a94f489e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"1e189a19fae505de745c69dbb168a51fe98419c9e11747f4f79c161c6db4e7ca","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000abec20\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-85mvxbzslwt7\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b45304ce-67dd-49f1-b5d1-6815f1b21d81\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b45304ce-67dd-49f1-b5d1-6815f1b21d81\",\n \"name\": \"b45304ce-67dd-49f1-b5d1-6815f1b21d81\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:20.392806615Z\",\n \"endTime\": \"2026-09-22T17:05:24.147500124Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000abec20\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-85mvxbzslwt7\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b45304ce-67dd-49f1-b5d1-6815f1b21d81\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b45304ce-67dd-49f1-b5d1-6815f1b21d81\",\n \"name\": \"b45304ce-67dd-49f1-b5d1-6815f1b21d81\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:20.392806615Z\",\n \"endTime\": \"2026-09-22T17:05:24.147500124Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"60d6d54bd256ca68907cf875e9e5b2ebd24f9e57cc0f3ff9b846f65e95320cbb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"8ed34e522350ffda7fb250647bec9f130736688154ac90808bd25676fa118f78","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001560120\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-j5gj726jttgj\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63296de6-e4f7-479c-8d94-e247b3fb7570\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63296de6-e4f7-479c-8d94-e247b3fb7570\",\n \"name\": \"63296de6-e4f7-479c-8d94-e247b3fb7570\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:11.224909027Z\",\n \"endTime\": \"2026-09-22T17:05:14.704490055Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001560120\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-j5gj726jttgj\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63296de6-e4f7-479c-8d94-e247b3fb7570\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63296de6-e4f7-479c-8d94-e247b3fb7570\",\n \"name\": \"63296de6-e4f7-479c-8d94-e247b3fb7570\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:11.224909027Z\",\n \"endTime\": \"2026-09-22T17:05:14.704490055Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6389ce96776e0e48b5896ce36fa1fcb048838a923563f68e03530a61a3469e41","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"804139726f070b1041f8100fde4c47aa1c25e4dfc0c87f55286bfe671852281e","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010f5240\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-89zmnzc9kp4x\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4644d955-2d8e-4777-9707-0e2d24d0ad08\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4644d955-2d8e-4777-9707-0e2d24d0ad08\",\n \"name\": \"4644d955-2d8e-4777-9707-0e2d24d0ad08\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:11.010189501Z\",\n \"endTime\": \"2026-09-22T17:05:14.812991744Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010f5240\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-89zmnzc9kp4x\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4644d955-2d8e-4777-9707-0e2d24d0ad08\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4644d955-2d8e-4777-9707-0e2d24d0ad08\",\n \"name\": \"4644d955-2d8e-4777-9707-0e2d24d0ad08\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:11.010189501Z\",\n \"endTime\": \"2026-09-22T17:05:14.812991744Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"641afff12bfae96610b40caec3450fa0fc605eac9108969b5208476fcc14f1dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"b5b9ace7c0308fb8f16c98cc948db5775855e4b1dcffad726de65815e01eaa00","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-wfcd7d78hw84/cluster-candidate-5-0-7q76df should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bbe7a0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-7q76df: failed waiting for cluster=\"cluster-candidate-5-0-7q76df\" in resourcegroup=\"rg-candidate-5-0-wfcd7d78hw84\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6972e50c-f532-4d25-a01e-ed0f0fc64509\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6972e50c-f532-4d25-a01e-ed0f0fc64509\",\n \"name\": \"6972e50c-f532-4d25-a01e-ed0f0fc64509\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:22.06647716Z\",\n \"endTime\": \"2026-09-22T17:05:24.024340059Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-wfcd7d78hw84/cluster-candidate-5-0-7q76df should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bbe7a0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-7q76df: failed waiting for cluster=\"cluster-candidate-5-0-7q76df\" in resourcegroup=\"rg-candidate-5-0-wfcd7d78hw84\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6972e50c-f532-4d25-a01e-ed0f0fc64509\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6972e50c-f532-4d25-a01e-ed0f0fc64509\",\n \"name\": \"6972e50c-f532-4d25-a01e-ed0f0fc64509\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:22.06647716Z\",\n \"endTime\": \"2026-09-22T17:05:24.024340059Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6ce4995bb4b91ba0940d08187cc10481e926c41f267ba8fc8f8f514f6f16f0bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"eace06cfe41d2ebbae3c6b22326ca19f7591d166aa15abe4fec7bfa0baabb41b","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-qsl4cr\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001056760\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-qsl4cr: failed waiting for cluster=\"cp-ystream-upgrade-4-22-qsl4cr\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-zvshks8mrjrp\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f41e8e09-6185-4b75-9296-bd49e5530f90\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f41e8e09-6185-4b75-9296-bd49e5530f90\",\n \"name\": \"f41e8e09-6185-4b75-9296-bd49e5530f90\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:19.692646834Z\",\n \"endTime\": \"2026-09-22T17:05:24.125571007Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-qsl4cr\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001056760\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-qsl4cr: failed waiting for cluster=\"cp-ystream-upgrade-4-22-qsl4cr\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-zvshks8mrjrp\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f41e8e09-6185-4b75-9296-bd49e5530f90\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f41e8e09-6185-4b75-9296-bd49e5530f90\",\n \"name\": \"f41e8e09-6185-4b75-9296-bd49e5530f90\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:19.692646834Z\",\n \"endTime\": \"2026-09-22T17:05:24.125571007Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6fc7542c4afb034d7be12a3a8b687cdeeabaa2aa4f8d7691240ef234b99aec68","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"alert","signature_id":"e94d5731f1171ed8ccd1a80a6bd7d03994a9e3251c8ef2f3a3e39c2a85a34276","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors6h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T17:38:22Z\n Severity: Sev3\n Labels: alertname=\"userJourneyClusterProvisionErrors6h30m\", cluster=\"ci01-j4475392-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4475392/providers/microsoft.monitor/accounts/services-j4475392\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n Description: More than 30% of completed cluster create (install) operations on ci01-j4475392-svc failed over the last 6 hours (at least 5 completions), a 6x burn of the 95% SLO budget.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d11e817743c9b7b223c4f7499ea10417bad7afb08d112eac38001716535808b0","failure_pattern":"alert [svc] userJourneyClusterProvisionErrors6h30m fired","search_query":"alert [svc] userJourneyClusterProvisionErrors6h30m fired"}},{"row_id":"76ac71c5a430e6eea7f07243b27db1de29b6e793be26e3c880567249f3ef11f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"a19b9c1a10e185f952d6f4d9f82ce909f40d51f7f31c07bb14777fb31726a16f","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f24e20\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-m6tl2cdt492z\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1abffcc9-0969-4c8e-9169-58a7701f57c8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1abffcc9-0969-4c8e-9169-58a7701f57c8\",\n \"name\": \"1abffcc9-0969-4c8e-9169-58a7701f57c8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:37:10.047595495Z\",\n \"endTime\": \"2026-09-22T17:06:14.624098103Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f24e20\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-m6tl2cdt492z\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1abffcc9-0969-4c8e-9169-58a7701f57c8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1abffcc9-0969-4c8e-9169-58a7701f57c8\",\n \"name\": \"1abffcc9-0969-4c8e-9169-58a7701f57c8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:37:10.047595495Z\",\n \"endTime\": \"2026-09-22T17:06:14.624098103Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7f6ee0edc3fafff81b4385b71059888583a21b9797ef014821c320a6b5be44d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"58aac2aff6f1ae29ed7c65f0b97e2df1f1fdd10c28f77f899c5d1127d0967e40","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e78f20\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-xndn9c5dzjz9\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e9538312-829d-4299-a8c1-299b80ee3f61\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e9538312-829d-4299-a8c1-299b80ee3f61\",\n \"name\": \"e9538312-829d-4299-a8c1-299b80ee3f61\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T17:11:29.127000193Z\",\n \"endTime\": \"2026-09-22T17:40:29.489392482Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e78f20\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-xndn9c5dzjz9\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e9538312-829d-4299-a8c1-299b80ee3f61\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e9538312-829d-4299-a8c1-299b80ee3f61\",\n \"name\": \"e9538312-829d-4299-a8c1-299b80ee3f61\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T17:11:29.127000193Z\",\n \"endTime\": \"2026-09-22T17:40:29.489392482Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8b9b4061349d27fc4a915380e208659bfa9a16951de6e49df4c9300b379996e6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"3702443bad4883b0bd4d8e1d16f90d0834ee6fdc57f03270085d8eaadca8af58","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001188420\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-2njgkd7pf75x\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56b97f7e-7917-424a-8feb-3bbf833220a6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56b97f7e-7917-424a-8feb-3bbf833220a6\",\n \"name\": \"56b97f7e-7917-424a-8feb-3bbf833220a6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:35:51.739370447Z\",\n \"endTime\": \"2026-09-22T17:04:54.27229918Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001188420\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-2njgkd7pf75x\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56b97f7e-7917-424a-8feb-3bbf833220a6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56b97f7e-7917-424a-8feb-3bbf833220a6\",\n \"name\": \"56b97f7e-7917-424a-8feb-3bbf833220a6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:35:51.739370447Z\",\n \"endTime\": \"2026-09-22T17:04:54.27229918Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8fd56f2143eded45b2a4d9b15e2183b9f77b20b0581b924253acfd97fa03190a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"b149cb73141f2a05314c417a812e2beec7368d22e86ab78a853852e548e0e065","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008c05e0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-q22cpt6sfmdd\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/42a9d035-c8f7-4515-96cd-4a75bad14685\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/42a9d035-c8f7-4515-96cd-4a75bad14685\",\n \"name\": \"42a9d035-c8f7-4515-96cd-4a75bad14685\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:26.805037234Z\",\n \"endTime\": \"2026-09-22T17:05:34.371586074Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008c05e0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-q22cpt6sfmdd\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/42a9d035-c8f7-4515-96cd-4a75bad14685\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/42a9d035-c8f7-4515-96cd-4a75bad14685\",\n \"name\": \"42a9d035-c8f7-4515-96cd-4a75bad14685\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:26.805037234Z\",\n \"endTime\": \"2026-09-22T17:05:34.371586074Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"95bb4b2bdd102b5d6a939482b590b2732081c5674a5ce0094d97978daf4ad99d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"58f2b440a994a0c878d24293dff5f6b4514980f374d184722cf27fc196af816f","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-46r9kp\" with version 4.21.33 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a72500\u003e: \n failed to create HCP cluster cluster-zstream-4-21-46r9kp: failed waiting for cluster=\"cluster-zstream-4-21-46r9kp\" in resourcegroup=\"rg-zstream-upgrade-4-21-kqtbx7l8f2cd\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c40ffce-91f8-432f-98a9-a17f456386e1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c40ffce-91f8-432f-98a9-a17f456386e1\",\n \"name\": \"7c40ffce-91f8-432f-98a9-a17f456386e1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:11.107965605Z\",\n \"endTime\": \"2026-09-22T17:05:14.587756636Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-46r9kp\" with version 4.21.33 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a72500\u003e: \n failed to create HCP cluster cluster-zstream-4-21-46r9kp: failed waiting for cluster=\"cluster-zstream-4-21-46r9kp\" in resourcegroup=\"rg-zstream-upgrade-4-21-kqtbx7l8f2cd\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c40ffce-91f8-432f-98a9-a17f456386e1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c40ffce-91f8-432f-98a9-a17f456386e1\",\n \"name\": \"7c40ffce-91f8-432f-98a9-a17f456386e1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:11.107965605Z\",\n \"endTime\": \"2026-09-22T17:05:14.587756636Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9b966360321f6d4007270973aa569f9c0fe03120c7453c630fc6a44b112d2ed2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"6816e0ef7bc04a15cb8eef8e32843b61c4fcdf6e64b22c55680deaf9f1aee076","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-7mgdpk at version 4.22.14\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0e240\u003e: \n failed to create HCP cluster np-skip-minor-7mgdpk: failed waiting for cluster=\"np-skip-minor-7mgdpk\" in resourcegroup=\"rg-np-skip-minor-7mgdpk-zppwbmg2pxz9\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d63e95f6-06b8-4ecd-adcb-ca227f3cf447\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d63e95f6-06b8-4ecd-adcb-ca227f3cf447\",\n \"name\": \"d63e95f6-06b8-4ecd-adcb-ca227f3cf447\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:28.13810987Z\",\n \"endTime\": \"2026-09-22T17:05:34.35992663Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-7mgdpk at version 4.22.14\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0e240\u003e: \n failed to create HCP cluster np-skip-minor-7mgdpk: failed waiting for cluster=\"np-skip-minor-7mgdpk\" in resourcegroup=\"rg-np-skip-minor-7mgdpk-zppwbmg2pxz9\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d63e95f6-06b8-4ecd-adcb-ca227f3cf447\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d63e95f6-06b8-4ecd-adcb-ca227f3cf447\",\n \"name\": \"d63e95f6-06b8-4ecd-adcb-ca227f3cf447\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:28.13810987Z\",\n \"endTime\": \"2026-09-22T17:05:34.35992663Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9f0326e5b38ded2aa1f103aa935284686c342a256005a75483993d7481d75c70","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"7aa73d22a62996df337f4919aa5b6a2b148aca5d4d6e3c14a3ddcf0f290454af","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000afc5e0\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-km6krsmrn4pb\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1f22da8f-3a4b-46a6-9028-2313648c12c9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1f22da8f-3a4b-46a6-9028-2313648c12c9\",\n \"name\": \"1f22da8f-3a4b-46a6-9028-2313648c12c9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:15.314214518Z\",\n \"endTime\": \"2026-09-22T17:05:24.094562064Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000afc5e0\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-km6krsmrn4pb\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1f22da8f-3a4b-46a6-9028-2313648c12c9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1f22da8f-3a4b-46a6-9028-2313648c12c9\",\n \"name\": \"1f22da8f-3a4b-46a6-9028-2313648c12c9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:15.314214518Z\",\n \"endTime\": \"2026-09-22T17:05:24.094562064Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a1d9936c123a4ee253f29b0e4ce5f596a05df3816e2cdd7f77b38f09f9ffafe2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"623a2d85a1841b8bf34570ecee70c8bdbeff7f363469e44c0648673b157343af","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--q8pfz7dcxcwn/cluster-hs-pre-6s5mlg should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00110ee80\u003e: \n failed to create HCP cluster cluster-hs-pre-6s5mlg: failed waiting for cluster=\"cluster-hs-pre-6s5mlg\" in resourcegroup=\"rg-hs-pre--q8pfz7dcxcwn\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/953251c4-2f9e-4fc4-8a41-b0e88bd1dfe6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/953251c4-2f9e-4fc4-8a41-b0e88bd1dfe6\",\n \"name\": \"953251c4-2f9e-4fc4-8a41-b0e88bd1dfe6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:11.476929134Z\",\n \"endTime\": \"2026-09-22T17:05:14.534373591Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--q8pfz7dcxcwn/cluster-hs-pre-6s5mlg should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00110ee80\u003e: \n failed to create HCP cluster cluster-hs-pre-6s5mlg: failed waiting for cluster=\"cluster-hs-pre-6s5mlg\" in resourcegroup=\"rg-hs-pre--q8pfz7dcxcwn\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/953251c4-2f9e-4fc4-8a41-b0e88bd1dfe6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/953251c4-2f9e-4fc4-8a41-b0e88bd1dfe6\",\n \"name\": \"953251c4-2f9e-4fc4-8a41-b0e88bd1dfe6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:11.476929134Z\",\n \"endTime\": \"2026-09-22T17:05:14.534373591Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a2ba884fefc693fd1497a78bfdad3f2ca60f6ad2125a264a1b196598d87cf669","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"fb9c548889e979624bf5ffaf1b274586a10ce39b24af64381e4f94d6371d35e7","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-gb92sf with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ac37a0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-gb92sf: failed waiting for cluster=\"np-version-upgrade-cluster-gb92sf\" in resourcegroup=\"rg-np-version-upgrade-gb92sf-xwrmnplklbpq\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/99bfd1c8-1de8-4bb4-9f76-bdbfa324ac04\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/99bfd1c8-1de8-4bb4-9f76-bdbfa324ac04\",\n \"name\": \"99bfd1c8-1de8-4bb4-9f76-bdbfa324ac04\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:34.388273814Z\",\n \"endTime\": \"2026-09-22T17:05:34.376441059Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-gb92sf with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ac37a0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-gb92sf: failed waiting for cluster=\"np-version-upgrade-cluster-gb92sf\" in resourcegroup=\"rg-np-version-upgrade-gb92sf-xwrmnplklbpq\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/99bfd1c8-1de8-4bb4-9f76-bdbfa324ac04\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/99bfd1c8-1de8-4bb4-9f76-bdbfa324ac04\",\n \"name\": \"99bfd1c8-1de8-4bb4-9f76-bdbfa324ac04\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:34.388273814Z\",\n \"endTime\": \"2026-09-22T17:05:34.376441059Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a31a4470e5ea87887b1b2ea531fc5034820ac1230e15ba2367e5729b1db1d3e1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"1c3157c947f58fd7bff96c9f69e32f40095840b97dafe53456bc1253bd4040ee","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d9f1a0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-kgwvsmk5rm2h\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6993d29-caec-456b-8254-690fe28ba79d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6993d29-caec-456b-8254-690fe28ba79d\",\n \"name\": \"a6993d29-caec-456b-8254-690fe28ba79d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:27.150078447Z\",\n \"endTime\": \"2026-09-22T17:05:34.293937324Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d9f1a0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-kgwvsmk5rm2h\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6993d29-caec-456b-8254-690fe28ba79d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6993d29-caec-456b-8254-690fe28ba79d\",\n \"name\": \"a6993d29-caec-456b-8254-690fe28ba79d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:27.150078447Z\",\n \"endTime\": \"2026-09-22T17:05:34.293937324Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a413188280d20375f64708a89ea7757b0e3fb6ec1e4193ad7041b0ac8bcf7812","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"6d0286b4a77d24ef7787809bf863c3ac7059e4e55a1c01eb88019f4277fde825","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001086f20\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-s4xt9ndfdfqn\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95ac10fd-a19e-4b3d-a436-7176e2fd4f08\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95ac10fd-a19e-4b3d-a436-7176e2fd4f08\",\n \"name\": \"95ac10fd-a19e-4b3d-a436-7176e2fd4f08\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:38:42.665905872Z\",\n \"endTime\": \"2026-09-22T17:07:49.360946035Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001086f20\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-s4xt9ndfdfqn\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95ac10fd-a19e-4b3d-a436-7176e2fd4f08\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95ac10fd-a19e-4b3d-a436-7176e2fd4f08\",\n \"name\": \"95ac10fd-a19e-4b3d-a436-7176e2fd4f08\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:38:42.665905872Z\",\n \"endTime\": \"2026-09-22T17:07:49.360946035Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b592101708d44c418af93aa8ae3c5b5359cb7cbfa5c8cf0d4369cda3fbe707fb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"9232a7400391f356316f15eeb51fcd91beb1f25acba3ab886fce2c7e5b56d1c1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-mmf24lvbl69k/cluster-candidate-4-22-jq772f should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001116360\u003e: \n failed to create HCP cluster cluster-candidate-4-22-jq772f: failed waiting for cluster=\"cluster-candidate-4-22-jq772f\" in resourcegroup=\"rg-candidate-4-22-mmf24lvbl69k\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d37ba33c-41ef-4bc9-b99d-39c30ef70d7f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d37ba33c-41ef-4bc9-b99d-39c30ef70d7f\",\n \"name\": \"d37ba33c-41ef-4bc9-b99d-39c30ef70d7f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:11.574580339Z\",\n \"endTime\": \"2026-09-22T17:05:15.304759832Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-mmf24lvbl69k/cluster-candidate-4-22-jq772f should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001116360\u003e: \n failed to create HCP cluster cluster-candidate-4-22-jq772f: failed waiting for cluster=\"cluster-candidate-4-22-jq772f\" in resourcegroup=\"rg-candidate-4-22-mmf24lvbl69k\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d37ba33c-41ef-4bc9-b99d-39c30ef70d7f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d37ba33c-41ef-4bc9-b99d-39c30ef70d7f\",\n \"name\": \"d37ba33c-41ef-4bc9-b99d-39c30ef70d7f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:11.574580339Z\",\n \"endTime\": \"2026-09-22T17:05:15.304759832Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b84fa852010b1ede6f0080a32a279fb5c33d269427961fc93fec7df54930ce47","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"alert","signature_id":"72e6ccb9b356ddbe42696b63bd109e76bb2f91fafa56b87cf5cdf1bfa7e627b3","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T17:49:51Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j4475392-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-ff8gg\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-ff8gg (namespace arobit) on cluster ci01-j4475392-svc has averaged 123.8% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"ba3b9ca6bb34ed9cff6213a0568b896052f6bde326003552f645acadd5eedbfc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"fbff5fde4987fdb4c2873cc854520923ed13f84351aeb3493ea93be60c17edb5","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014b0700\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-cth475kbhvnz\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ad99362b-03f1-478d-8c68-903b1a5b453e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ad99362b-03f1-478d-8c68-903b1a5b453e\",\n \"name\": \"ad99362b-03f1-478d-8c68-903b1a5b453e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:15.474739273Z\",\n \"endTime\": \"2026-09-22T17:05:24.196969347Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014b0700\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-cth475kbhvnz\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ad99362b-03f1-478d-8c68-903b1a5b453e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ad99362b-03f1-478d-8c68-903b1a5b453e\",\n \"name\": \"ad99362b-03f1-478d-8c68-903b1a5b453e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:15.474739273Z\",\n \"endTime\": \"2026-09-22T17:05:24.196969347Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bae44ed664984d6a80d76ebb2f404d2c77ebbf2d64ac0e3347c02ae8ac63fad0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"096fb94a6c94af770de1adc257d2294c1a5548271d7b9aad906d0c0a59e70fff","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:83]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c521c0\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-8ql5kd5f7tjm\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/547703e0-0d1c-43cc-a185-feb70bc56c33\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/547703e0-0d1c-43cc-a185-feb70bc56c33\",\n \"name\": \"547703e0-0d1c-43cc-a185-feb70bc56c33\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:27.4357733Z\",\n \"endTime\": \"2026-09-22T17:05:34.221694646Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:83]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c521c0\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-8ql5kd5f7tjm\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/547703e0-0d1c-43cc-a185-feb70bc56c33\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/547703e0-0d1c-43cc-a185-feb70bc56c33\",\n \"name\": \"547703e0-0d1c-43cc-a185-feb70bc56c33\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:27.4357733Z\",\n \"endTime\": \"2026-09-22T17:05:34.221694646Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c158a3111cf93dd7d33e917aaaedc0d44bf4e6aef3a64e6a069e3243cc064b4a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"c8c1a15166f9304465eaca377226a0779617f87f57e8d7a65ecbe7dd9d745a90","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fc8a80\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-rv2rq259ntc8\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/94923f6a-6919-48c0-80e9-3c3b9c3730d1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/94923f6a-6919-48c0-80e9-3c3b9c3730d1\",\n \"name\": \"94923f6a-6919-48c0-80e9-3c3b9c3730d1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:35:56.097535459Z\",\n \"endTime\": \"2026-09-22T17:05:04.00508538Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fc8a80\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-rv2rq259ntc8\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/94923f6a-6919-48c0-80e9-3c3b9c3730d1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/94923f6a-6919-48c0-80e9-3c3b9c3730d1\",\n \"name\": \"94923f6a-6919-48c0-80e9-3c3b9c3730d1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:35:56.097535459Z\",\n \"endTime\": \"2026-09-22T17:05:04.00508538Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c47d3aa68f5eb5e9ea562b766e4f78496607b81fdfc232b66f1ee5c88f60aa79","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"0b2a132ab1552649222c76502236db8e74948cddd63f8f30c41982bda0fd9971","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-r2ls78\" with version 4.22.13 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b566a0\u003e: \n failed to create HCP cluster cluster-zstream-4-22-r2ls78: failed waiting for cluster=\"cluster-zstream-4-22-r2ls78\" in resourcegroup=\"rg-zstream-upgrade-4-22-sm79xt4bwmhx\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/55386486-26be-4228-bb22-1d85147aacf3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/55386486-26be-4228-bb22-1d85147aacf3\",\n \"name\": \"55386486-26be-4228-bb22-1d85147aacf3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:16.148910931Z\",\n \"endTime\": \"2026-09-22T17:05:24.221837381Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-r2ls78\" with version 4.22.13 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b566a0\u003e: \n failed to create HCP cluster cluster-zstream-4-22-r2ls78: failed waiting for cluster=\"cluster-zstream-4-22-r2ls78\" in resourcegroup=\"rg-zstream-upgrade-4-22-sm79xt4bwmhx\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/55386486-26be-4228-bb22-1d85147aacf3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/55386486-26be-4228-bb22-1d85147aacf3\",\n \"name\": \"55386486-26be-4228-bb22-1d85147aacf3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:16.148910931Z\",\n \"endTime\": \"2026-09-22T17:05:24.221837381Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c8d491f1c6657346028b5411de8f1c752c03f830535634776073da670b64b015","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"b2ca19944d899fbd5a1416f0e4d1be6345d0d373d8de61d38fe13164fd82111f","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-b7bkwk\" with version 4.20.38 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00093c5c0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-b7bkwk: failed waiting for cluster=\"cluster-zstream-4-20-b7bkwk\" in resourcegroup=\"rg-zstream-upgrade-4-20-j6h5fx5bsw4s\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01b04d32-39e2-487e-ae84-b4c3e4359648\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01b04d32-39e2-487e-ae84-b4c3e4359648\",\n \"name\": \"01b04d32-39e2-487e-ae84-b4c3e4359648\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:21.363686989Z\",\n \"endTime\": \"2026-09-22T17:05:24.227617931Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-b7bkwk\" with version 4.20.38 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00093c5c0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-b7bkwk: failed waiting for cluster=\"cluster-zstream-4-20-b7bkwk\" in resourcegroup=\"rg-zstream-upgrade-4-20-j6h5fx5bsw4s\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01b04d32-39e2-487e-ae84-b4c3e4359648\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01b04d32-39e2-487e-ae84-b4c3e4359648\",\n \"name\": \"01b04d32-39e2-487e-ae84-b4c3e4359648\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:21.363686989Z\",\n \"endTime\": \"2026-09-22T17:05:24.227617931Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ce65792dcda3746d651ce157bd3892bf683822c551091864abb82483bc01b2ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"dd91c26e3cd9c656ccb1a1a45547d7332939e80b90ce720939d4434685892e6f","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a425a0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-knnjcpzbn8r2\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb18f2ee-8f38-48b6-9d86-84b032732a97\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb18f2ee-8f38-48b6-9d86-84b032732a97\",\n \"name\": \"cb18f2ee-8f38-48b6-9d86-84b032732a97\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:37.699532367Z\",\n \"endTime\": \"2026-09-22T17:05:44.628988758Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a425a0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-knnjcpzbn8r2\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb18f2ee-8f38-48b6-9d86-84b032732a97\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb18f2ee-8f38-48b6-9d86-84b032732a97\",\n \"name\": \"cb18f2ee-8f38-48b6-9d86-84b032732a97\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:37.699532367Z\",\n \"endTime\": \"2026-09-22T17:05:44.628988758Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ce9750dabcd6a898266cf4d686f6094016ca7764063026e4966d4a33f14d920c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"cc144734f26b1d2b89daf0a0292b0f322bcea668bea3417d61ea7b6957d358f0","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ae0ce0\u003e: \n failed to create HCP cluster ea-listsfgcb9: failed waiting for cluster=\"ea-listsfgcb9\" in resourcegroup=\"ea-list-m8pl2s5stks2\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1e9b63b5-90b2-40fb-859e-8c4e24d4ec59\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1e9b63b5-90b2-40fb-859e-8c4e24d4ec59\",\n \"name\": \"1e9b63b5-90b2-40fb-859e-8c4e24d4ec59\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:20.544914102Z\",\n \"endTime\": \"2026-09-22T17:05:24.196772116Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ae0ce0\u003e: \n failed to create HCP cluster ea-listsfgcb9: failed waiting for cluster=\"ea-listsfgcb9\" in resourcegroup=\"ea-list-m8pl2s5stks2\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1e9b63b5-90b2-40fb-859e-8c4e24d4ec59\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1e9b63b5-90b2-40fb-859e-8c4e24d4ec59\",\n \"name\": \"1e9b63b5-90b2-40fb-859e-8c4e24d4ec59\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:20.544914102Z\",\n \"endTime\": \"2026-09-22T17:05:24.196772116Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ceced87d8ca9f868b398cb332f33603d6d9b70b0b4eb8135847bf9a6ec4ef4a4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"aaafc5d30addb61ff662830f1217bc76ce43a223d5f303ad8447f9d259c6bef9","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008fe6a0\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-xxj2hlgdfm4c\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2b9400a7-5197-4acd-b7f8-f4eb830bfc0b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2b9400a7-5197-4acd-b7f8-f4eb830bfc0b\",\n \"name\": \"2b9400a7-5197-4acd-b7f8-f4eb830bfc0b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:06.037503121Z\",\n \"endTime\": \"2026-09-22T17:05:15.290687493Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008fe6a0\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-xxj2hlgdfm4c\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2b9400a7-5197-4acd-b7f8-f4eb830bfc0b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2b9400a7-5197-4acd-b7f8-f4eb830bfc0b\",\n \"name\": \"2b9400a7-5197-4acd-b7f8-f4eb830bfc0b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:06.037503121Z\",\n \"endTime\": \"2026-09-22T17:05:15.290687493Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cfd948e770fd1393eac9f12d482b07133005a5e643da7abb93d53f256c147eba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"f6d6a168159422ba9c70117e70748b52c1c05f58d631c53185b407f05a9704ce","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00015a360\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-9q4d8jcrrwfp\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a2c8014a-b687-41fd-ace7-be97be13a6e4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a2c8014a-b687-41fd-ace7-be97be13a6e4\",\n \"name\": \"a2c8014a-b687-41fd-ace7-be97be13a6e4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:10.297297909Z\",\n \"endTime\": \"2026-09-22T17:05:14.590541785Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00015a360\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-9q4d8jcrrwfp\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a2c8014a-b687-41fd-ace7-be97be13a6e4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a2c8014a-b687-41fd-ace7-be97be13a6e4\",\n \"name\": \"a2c8014a-b687-41fd-ace7-be97be13a6e4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:10.297297909Z\",\n \"endTime\": \"2026-09-22T17:05:14.590541785Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d06bb610c850a31789daf7e791457585ebefde162855ecacbddf746dd0edaa6a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"aab74da0f5bf35c1456f1863c7d9823e709750f96f75dca49cc4b64a53f482fa","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-z2p8kx with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017febe0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-z2p8kx: failed waiting for cluster=\"np-version-upgrade-cluster-z2p8kx\" in resourcegroup=\"rg-np-version-upgrade-z2p8kx-6psgzckkqp6m\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc9376d9-c75f-49c8-b375-fda4aabeb8cf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc9376d9-c75f-49c8-b375-fda4aabeb8cf\",\n \"name\": \"bc9376d9-c75f-49c8-b375-fda4aabeb8cf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:27.960631712Z\",\n \"endTime\": \"2026-09-22T17:05:34.274798638Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-z2p8kx with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017febe0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-z2p8kx: failed waiting for cluster=\"np-version-upgrade-cluster-z2p8kx\" in resourcegroup=\"rg-np-version-upgrade-z2p8kx-6psgzckkqp6m\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc9376d9-c75f-49c8-b375-fda4aabeb8cf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc9376d9-c75f-49c8-b375-fda4aabeb8cf\",\n \"name\": \"bc9376d9-c75f-49c8-b375-fda4aabeb8cf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:27.960631712Z\",\n \"endTime\": \"2026-09-22T17:05:34.274798638Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d0ce83ed0e02a6183b2ebae36eebc4d655fda802fd97ec1e02054254db3e5ddc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"457a5314345082c90ad67356518bb9d84bd8d68c40ec7946a1e92e67744e5389","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000af31d0\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-r6xxwmvr4vzb\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-r6xxwmvr4vzb, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000af31d0\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-r6xxwmvr4vzb\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-r6xxwmvr4vzb, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-57bkrhmpnrrx\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-57bkrhmpnrrx, error: context deadline exceeded"}},{"row_id":"d5ba324b82197a364b1303b4adb99098dc0769d27005acc25305ff2e2e188fc8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"a1f111f15dc32b37e44ac7b8ec2b54927129d700f72b123fbd0163455608e4ab","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e5a440\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-dhfmcr22g2ft\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a322f570-1cad-47c7-8044-b71331609113\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a322f570-1cad-47c7-8044-b71331609113\",\n \"name\": \"a322f570-1cad-47c7-8044-b71331609113\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:10.925233852Z\",\n \"endTime\": \"2026-09-22T17:05:15.583156498Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e5a440\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-dhfmcr22g2ft\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a322f570-1cad-47c7-8044-b71331609113\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a322f570-1cad-47c7-8044-b71331609113\",\n \"name\": \"a322f570-1cad-47c7-8044-b71331609113\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:10.925233852Z\",\n \"endTime\": \"2026-09-22T17:05:15.583156498Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"db185c65f3d316cda03aca1ac3e29296676db8f4677760f51e499b421df4bdbb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"2e05c40aeb58cbc76cefbe3d60e5dd6051b60dca39a8e9fafcc5368faeafc26d","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-5g8jcf\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e32c60\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-5g8jcf: failed waiting for cluster=\"cp-ystream-upgrade-4-21-5g8jcf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-6hsjb89rx8zt\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31b29abc-8d4e-4d29-ac99-57c16217a5ef\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31b29abc-8d4e-4d29-ac99-57c16217a5ef\",\n \"name\": \"31b29abc-8d4e-4d29-ac99-57c16217a5ef\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:18.914832024Z\",\n \"endTime\": \"2026-09-22T17:05:24.213170431Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-5g8jcf\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e32c60\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-5g8jcf: failed waiting for cluster=\"cp-ystream-upgrade-4-21-5g8jcf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-6hsjb89rx8zt\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31b29abc-8d4e-4d29-ac99-57c16217a5ef\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31b29abc-8d4e-4d29-ac99-57c16217a5ef\",\n \"name\": \"31b29abc-8d4e-4d29-ac99-57c16217a5ef\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:18.914832024Z\",\n \"endTime\": \"2026-09-22T17:05:24.213170431Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"defc7d52bb434aaf9d1fc27eeb83be3fe41a073b6f69af03c500657ccf92aba0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"6fb962141bed0f910461aa1150d3a12d1c1dcf673c5c5a69cbd92ff5968cdb05","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-lktj2c\" with version 5.0.0-rc.2 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010e2d40\u003e: \n failed to create HCP cluster cluster-zstream-5-0-lktj2c: failed waiting for cluster=\"cluster-zstream-5-0-lktj2c\" in resourcegroup=\"rg-zstream-upgrade-5-0-nwzl7cn6kqp6\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8b19ecd1-cdce-44ce-9718-67cf7c27e00a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8b19ecd1-cdce-44ce-9718-67cf7c27e00a\",\n \"name\": \"8b19ecd1-cdce-44ce-9718-67cf7c27e00a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:22.485113932Z\",\n \"endTime\": \"2026-09-22T17:05:24.233153854Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-lktj2c\" with version 5.0.0-rc.2 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010e2d40\u003e: \n failed to create HCP cluster cluster-zstream-5-0-lktj2c: failed waiting for cluster=\"cluster-zstream-5-0-lktj2c\" in resourcegroup=\"rg-zstream-upgrade-5-0-nwzl7cn6kqp6\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8b19ecd1-cdce-44ce-9718-67cf7c27e00a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8b19ecd1-cdce-44ce-9718-67cf7c27e00a\",\n \"name\": \"8b19ecd1-cdce-44ce-9718-67cf7c27e00a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:22.485113932Z\",\n \"endTime\": \"2026-09-22T17:05:24.233153854Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e1cf956703512f296ccb63420e56b56ee6e84ff2dd14293cd234169e5c3f2808","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"alert","signature_id":"32e03851f37111fa5a94d086994b49ba0284bc78df8fdff2b940cf9240caab63","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors1h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T17:14:23Z\n Severity: Sev3\n Labels: alertname=\"userJourneyClusterProvisionErrors1h5m\", cluster=\"ci01-j4475392-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4475392/providers/microsoft.monitor/accounts/services-j4475392\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"cluster-provision-errors\"\n Description: More than 72% of completed cluster create (install) operations on ci01-j4475392-svc failed over the last hour with at least 3 failures, a 14.4x burn of the 95% SLO budget.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9a8854d173302655314e0bdcbc8411f9e276ffe0aadf4e25f0e0bad8f8dcb51d","failure_pattern":"alert [svc] userJourneyClusterProvisionErrors1h5m fired","search_query":"alert [svc] userJourneyClusterProvisionErrors1h5m fired"}},{"row_id":"e730570c4f781cd40187fd263cccddbfc1f06e19a2b6e907b2c4bf1209c375dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"595498bacd4e938bd044f59e576c914a426f362c531efc177db2352d292b3855","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-66g59p\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d3ea40\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-66g59p: failed waiting for cluster=\"cp-ystream-upgrade-5-0-66g59p\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-dmnztktbvcnh\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/89a5b6f9-6cb3-484b-b3ee-cf482e31ecb8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/89a5b6f9-6cb3-484b-b3ee-cf482e31ecb8\",\n \"name\": \"89a5b6f9-6cb3-484b-b3ee-cf482e31ecb8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:18.511400811Z\",\n \"endTime\": \"2026-09-22T17:05:24.116275999Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-66g59p\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d3ea40\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-66g59p: failed waiting for cluster=\"cp-ystream-upgrade-5-0-66g59p\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-dmnztktbvcnh\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/89a5b6f9-6cb3-484b-b3ee-cf482e31ecb8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/89a5b6f9-6cb3-484b-b3ee-cf482e31ecb8\",\n \"name\": \"89a5b6f9-6cb3-484b-b3ee-cf482e31ecb8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:18.511400811Z\",\n \"endTime\": \"2026-09-22T17:05:24.116275999Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f385b733dd5feed921872e683659d043d21e07c7087dab54ec4caebee7be25c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"5202c14b8a67b24502a01693eb0e6502a9081428fcd7886134dd25c4ebf94bdc","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f670a0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-kcthj55945nv\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/591c7311-7de3-4e2e-b8bb-fbe0b4624e66\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/591c7311-7de3-4e2e-b8bb-fbe0b4624e66\",\n \"name\": \"591c7311-7de3-4e2e-b8bb-fbe0b4624e66\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:37:05.62592778Z\",\n \"endTime\": \"2026-09-22T17:06:14.174276347Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f670a0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-kcthj55945nv\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/591c7311-7de3-4e2e-b8bb-fbe0b4624e66\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/591c7311-7de3-4e2e-b8bb-fbe0b4624e66\",\n \"name\": \"591c7311-7de3-4e2e-b8bb-fbe0b4624e66\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:37:05.62592778Z\",\n \"endTime\": \"2026-09-22T17:06:14.174276347Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f4eee67d2501c82a594f9e0e83440813ad3eae447cc3fca463d93940fff32143","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"c9e77ba097967d05f7bfb10ea65f895976153a5d4d5aca8181798ec5188c475d","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ed4140\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-7f9fwfzj96zl\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/48240b99-a128-4809-98a9-f9a52f0f247e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/48240b99-a128-4809-98a9-f9a52f0f247e\",\n \"name\": \"48240b99-a128-4809-98a9-f9a52f0f247e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:22.143851357Z\",\n \"endTime\": \"2026-09-22T17:05:24.113006702Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ed4140\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-7f9fwfzj96zl\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/48240b99-a128-4809-98a9-f9a52f0f247e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/48240b99-a128-4809-98a9-f9a52f0f247e\",\n \"name\": \"48240b99-a128-4809-98a9-f9a52f0f247e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:22.143851357Z\",\n \"endTime\": \"2026-09-22T17:05:24.113006702Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f636fbc5148bab88d4a3e77dfa74ffb91fcb55bb6b1d03c7b387f15f8ac11bea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","failed_at":"e2e","signature_id":"a08aba7b10f89db3de8a4b9e00fac22f367da0ace6bdc0a9edfe1b4d61fc1984","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c14a40\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-62972sjbd4tf\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb15bb25-f3ed-4ac7-a122-ae29b89ccd00\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb15bb25-f3ed-4ac7-a122-ae29b89ccd00\",\n \"name\": \"eb15bb25-f3ed-4ac7-a122-ae29b89ccd00\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:20.790218421Z\",\n \"endTime\": \"2026-09-22T17:05:24.226195914Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c14a40\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-62972sjbd4tf\" to finish creating: GET https://rp.j4475392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb15bb25-f3ed-4ac7-a122-ae29b89ccd00\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb15bb25-f3ed-4ac7-a122-ae29b89ccd00\",\n \"name\": \"eb15bb25-f3ed-4ac7-a122-ae29b89ccd00\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:36:20.790218421Z\",\n \"endTime\": \"2026-09-22T17:05:24.226195914Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":63,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","9a8854d173302655314e0bdcbc8411f9e276ffe0aadf4e25f0e0bad8f8dcb51d","d11e817743c9b7b223c4f7499ea10417bad7afb08d112eac38001716535808b0","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":63,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102421926960959488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5846,"pr_state":"closed","pr_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","final_merged_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T15:37:03Z","started_at":"2026-09-22T15:37:03Z","completed_at":"2026-09-22T17:50:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T18:07:10.287024649Z","timing_metadata_checked_at":"2026-09-22T18:07:10.287024649Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T18:07:10.290533437Z","region_metadata_checked_at":"2026-09-22T18:07:10.290533437Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"2dca02a69647900b60b770a6c49a0b5796e2903005288c0778d8bf4a65ef6b41","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102421926960959488","occurred_at":"2026-09-22T15:37:03Z","failed_at":"e2e","signature_id":"454db31ed5c16068e777507735c5d53198406e84bb457480d945fb2e9b36f1e5","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a3e0f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-sxrknhrcf2sv\" to finish deleting: GET https://rp.j0959488.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d6a496d1-7d6a-46b3-b956-b999573cbcee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d6a496d1-7d6a-46b3-b956-b999573cbcee\",\n \"name\": \"d6a496d1-7d6a-46b3-b956-b999573cbcee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T17:10:29.991328675Z\",\n \"endTime\": \"2026-09-22T17:34:37.126574936Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t05q325vna98avgbb2eeekk2eu34dgn still exists (deletion dispatched at 2026-09-22T17:10:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a3e0f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-sxrknhrcf2sv\" to finish deleting: GET https://rp.j0959488.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d6a496d1-7d6a-46b3-b956-b999573cbcee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d6a496d1-7d6a-46b3-b956-b999573cbcee\",\n \"name\": \"d6a496d1-7d6a-46b3-b956-b999573cbcee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T17:10:29.991328675Z\",\n \"endTime\": \"2026-09-22T17:34:37.126574936Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t05q325vna98avgbb2eeekk2eu34dgn still exists (deletion dispatched at 2026-09-22T17:10:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"75b84e2446b4b16ed1a3f00107c48e2822b5f8f84413479678b53649aca0769a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102421926960959488","occurred_at":"2026-09-22T15:37:03Z","failed_at":"e2e","signature_id":"bea1828c4a4426018e92c0f20fa15d296358e1c6b1f1a42fec58d586a794f48c","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00177c1c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-wglcgf\" in resourcegroup=\"rg-np-version-upgrade-wglcgf-xqkdchgrl5rp\" to finish deleting: GET https://rp.j0959488.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/88d50275-408d-4ac0-97bc-7497d8fe6fd4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/88d50275-408d-4ac0-97bc-7497d8fe6fd4\",\n \"name\": \"88d50275-408d-4ac0-97bc-7497d8fe6fd4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T17:00:36.387615721Z\",\n \"endTime\": \"2026-09-22T17:24:37.112980505Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t05q51hjkkjt4r1accultpettv5d9l7 still exists (deletion dispatched at 2026-09-22T17:00:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00177c1c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-wglcgf\" in resourcegroup=\"rg-np-version-upgrade-wglcgf-xqkdchgrl5rp\" to finish deleting: GET https://rp.j0959488.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/88d50275-408d-4ac0-97bc-7497d8fe6fd4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/88d50275-408d-4ac0-97bc-7497d8fe6fd4\",\n \"name\": \"88d50275-408d-4ac0-97bc-7497d8fe6fd4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T17:00:36.387615721Z\",\n \"endTime\": \"2026-09-22T17:24:37.112980505Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t05q51hjkkjt4r1accultpettv5d9l7 still exists (deletion dispatched at 2026-09-22T17:00:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b2dc4b43731b57c378584b74a108ed779b23deadb71c9991c8e9c5698f0444b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102421926960959488","occurred_at":"2026-09-22T15:37:03Z","failed_at":"alert","signature_id":"2691f57bdab8f983b45857d3645b58fef98f444d625e7e014189bdfd9a9e4e78","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T17:28:44Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j0959488-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-kwb9z\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-kwb9z (namespace arobit) on cluster ci01-j0959488-svc has averaged 127.5% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"e58cc5a1a68405edb0af90928078d770e81cd30dd08c05a5a61f5652d681c633","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102421926960959488","occurred_at":"2026-09-22T15:37:03Z","failed_at":"e2e","signature_id":"cdf31af69c9f22027040e78d440beeb427fcf496c338987b154029c97938d2cf","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0018dd510\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0018dd510\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5493/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102420595500126208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5493,"pr_state":"open","pr_sha":"aa66f792b049b1f2bd70d208473fdddc00dbac7b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T15:31:45Z","started_at":"2026-09-22T15:31:45Z","completed_at":"2026-09-22T15:44:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T16:01:08.864529395Z","timing_metadata_checked_at":"2026-09-22T16:01:08.864529395Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-22T16:01:08.870429876Z","region_metadata_checked_at":"2026-09-22T17:04:02.232404025Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"21798cf308bc0d1c982310b779a687cf75005e4a578632c6c696940d4bfb1fca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5493/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102420595500126208","occurred_at":"2026-09-22T15:31:45Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412746732605440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6995,"pr_state":"closed","pr_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","final_merged_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T15:00:34Z","started_at":"2026-09-22T15:00:34Z","completed_at":"2026-09-22T17:17:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T17:25:39.681725902Z","timing_metadata_checked_at":"2026-09-22T17:25:39.681725902Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T17:25:39.685465189Z","region_metadata_checked_at":"2026-09-22T17:25:39.685465189Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"1103e5121c89e00fc6d748597da3d5e1e614b9a047e5180bd6953119c2b557ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412746732605440","occurred_at":"2026-09-22T15:00:34Z","failed_at":"e2e","signature_id":"182f151f4d335c3fa9f666fbf1c8637609ed53fb8725e376854f43c1b072f900","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0010f82a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-5s7lbj2gl95t\" to finish deleting: GET https://rp.j2605440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1f29ddf1-5f30-4805-8772-bb86c797c186\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1f29ddf1-5f30-4805-8772-bb86c797c186\",\n \"name\": \"1f29ddf1-5f30-4805-8772-bb86c797c186\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:32:22.534342212Z\",\n \"endTime\": \"2026-09-22T16:56:24.631597443Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t059ndd697kat3tga063u3hbor3fu8r still exists (deletion dispatched at 2026-09-22T16:32:24Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0010f82a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-5s7lbj2gl95t\" to finish deleting: GET https://rp.j2605440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1f29ddf1-5f30-4805-8772-bb86c797c186\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1f29ddf1-5f30-4805-8772-bb86c797c186\",\n \"name\": \"1f29ddf1-5f30-4805-8772-bb86c797c186\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:32:22.534342212Z\",\n \"endTime\": \"2026-09-22T16:56:24.631597443Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t059ndd697kat3tga063u3hbor3fu8r still exists (deletion dispatched at 2026-09-22T16:32:24Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"12085e217c1c5546f967e077cc4396dccfb62a517a7d30ebdba06fcca40bfc12","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412746732605440","occurred_at":"2026-09-22T15:00:34Z","failed_at":"alert","signature_id":"56de1d48b5dba8bd66ed5eba60745f0f8b40aa65ee4126e312322ff3f116fff4","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T16:22:37Z\n Ended: 2026-09-22T16:28:38Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j2605440-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j2605440-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"7362407ee747230eab4497132515fa1f2d54c0965c9d732fa1e9afddbc5c8764","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412746732605440","occurred_at":"2026-09-22T15:00:34Z","failed_at":"e2e","signature_id":"cf75714f4015843ed8f32a3f4098c758891337ab873b13195fdad1e87c280797","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0015f4378\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-9wppmrlqtvrm\" to finish deleting: GET https://rp.j2605440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/84a79aab-657b-4e08-be66-ba52c724c909\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/84a79aab-657b-4e08-be66-ba52c724c909\",\n \"name\": \"84a79aab-657b-4e08-be66-ba52c724c909\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:14:48.813812977Z\",\n \"endTime\": \"2026-09-22T16:38:51.572108159Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t05ad0e451o6jjr3d5o5rv4piihraka still exists (deletion dispatched at 2026-09-22T16:14:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0015f4378\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-9wppmrlqtvrm\" to finish deleting: GET https://rp.j2605440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/84a79aab-657b-4e08-be66-ba52c724c909\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/84a79aab-657b-4e08-be66-ba52c724c909\",\n \"name\": \"84a79aab-657b-4e08-be66-ba52c724c909\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:14:48.813812977Z\",\n \"endTime\": \"2026-09-22T16:38:51.572108159Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t05ad0e451o6jjr3d5o5rv4piihraka still exists (deletion dispatched at 2026-09-22T16:14:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8ee42282cc35a69e69f53a6857229e32928a7405a2dcb22692ce046f8debb678","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412746732605440","occurred_at":"2026-09-22T15:00:34Z","failed_at":"e2e","signature_id":"6dc54736f3b084b7ef7df392cad229a0edd37b47ab47338769672396e43cf0ff","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001f861f8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-wvtbxf\" in resourcegroup=\"rg-hs-pre--xzhjszx9k2c7\" to finish deleting: GET https://rp.j2605440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/82218643-5493-4a8c-b78b-cdeb6989d0fc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/82218643-5493-4a8c-b78b-cdeb6989d0fc\",\n \"name\": \"82218643-5493-4a8c-b78b-cdeb6989d0fc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:10:17.188585441Z\",\n \"endTime\": \"2026-09-22T16:34:21.604473698Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t059l0gr7df9ftcbt72ja7n96fahcrt still exists (deletion dispatched at 2026-09-22T16:10:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001f861f8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-wvtbxf\" in resourcegroup=\"rg-hs-pre--xzhjszx9k2c7\" to finish deleting: GET https://rp.j2605440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/82218643-5493-4a8c-b78b-cdeb6989d0fc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/82218643-5493-4a8c-b78b-cdeb6989d0fc\",\n \"name\": \"82218643-5493-4a8c-b78b-cdeb6989d0fc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:10:17.188585441Z\",\n \"endTime\": \"2026-09-22T16:34:21.604473698Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t059l0gr7df9ftcbt72ja7n96fahcrt still exists (deletion dispatched at 2026-09-22T16:10:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f3ac87b59284cd36361139e46c31ab3336ad5722db538f5a2c5ce863270aeb86","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412746732605440","occurred_at":"2026-09-22T15:00:34Z","failed_at":"e2e","signature_id":"a9df2cc0c774e75bc299fbf77563ccb4e81f10a214bbd561e63e891df2f8e874","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000cca6f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-9958km482jsv\" to finish deleting: GET https://rp.j2605440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/07560251-b40e-43d9-b428-54fa8fba0e71\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/07560251-b40e-43d9-b428-54fa8fba0e71\",\n \"name\": \"07560251-b40e-43d9-b428-54fa8fba0e71\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:22:28.013921275Z\",\n \"endTime\": \"2026-09-22T16:46:31.609629346Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t059hc0kkjcrvnnkosl8h62gegnbh6e still exists (deletion dispatched at 2026-09-22T16:22:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000cca6f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-9958km482jsv\" to finish deleting: GET https://rp.j2605440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/07560251-b40e-43d9-b428-54fa8fba0e71\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/07560251-b40e-43d9-b428-54fa8fba0e71\",\n \"name\": \"07560251-b40e-43d9-b428-54fa8fba0e71\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:22:28.013921275Z\",\n \"endTime\": \"2026-09-22T16:46:31.609629346Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t059hc0kkjcrvnnkosl8h62gegnbh6e still exists (deletion dispatched at 2026-09-22T16:22:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412736506892288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6756,"pr_state":"open","pr_sha":"a65a65d0f5af3365546d2f648ce8be93984b79b0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T15:00:31Z","started_at":"2026-09-22T15:00:31Z","completed_at":"2026-09-22T17:30:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T17:46:38.868405737Z","timing_metadata_checked_at":"2026-09-22T17:46:38.868405737Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T17:46:38.872156924Z","region_metadata_checked_at":"2026-09-22T17:46:38.872156924Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"3c2c4c98c85e800d19cb066add1fd025fc0c9c21a295937040eed69f95e07ec0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412736506892288","occurred_at":"2026-09-22T15:00:31Z","failed_at":"e2e","signature_id":"e421292d38ce67fa99edcfb49b7aec3948c57668458969aab7ac974ec6685308","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001828138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-hwpggw47v5hm\" to finish deleting: GET https://rp.j6892288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63edee93-2a7c-41df-a51b-a7f5f12c3550\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63edee93-2a7c-41df-a51b-a7f5f12c3550\",\n \"name\": \"63edee93-2a7c-41df-a51b-a7f5f12c3550\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:32:51.047517269Z\",\n \"endTime\": \"2026-09-22T16:56:56.822827308Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t05e8a37dph87bq1p0gjquml726f107 still exists (deletion dispatched at 2026-09-22T16:32:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001828138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-hwpggw47v5hm\" to finish deleting: GET https://rp.j6892288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63edee93-2a7c-41df-a51b-a7f5f12c3550\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/63edee93-2a7c-41df-a51b-a7f5f12c3550\",\n \"name\": \"63edee93-2a7c-41df-a51b-a7f5f12c3550\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:32:51.047517269Z\",\n \"endTime\": \"2026-09-22T16:56:56.822827308Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t05e8a37dph87bq1p0gjquml726f107 still exists (deletion dispatched at 2026-09-22T16:32:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"488cb3f9a662031773d803190616b96b761181c0caeeeef138275379879f749b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412736506892288","occurred_at":"2026-09-22T15:00:31Z","failed_at":"e2e","signature_id":"82a4812b53d7120071f99657652811f4dd24e212a34a1955fd32863186a418f1","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000d06288\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-n65mjwh6668r\" to finish deleting: GET https://rp.j6892288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5bce1036-3d2d-444f-ad5f-2938541cce95\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5bce1036-3d2d-444f-ad5f-2938541cce95\",\n \"name\": \"5bce1036-3d2d-444f-ad5f-2938541cce95\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:45:55.582699774Z\",\n \"endTime\": \"2026-09-22T17:09:56.885776378Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t05e9l3bms07pg766c7t42iidgiet8o still exists (deletion dispatched at 2026-09-22T16:45:57Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000d06288\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-n65mjwh6668r\" to finish deleting: GET https://rp.j6892288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5bce1036-3d2d-444f-ad5f-2938541cce95\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5bce1036-3d2d-444f-ad5f-2938541cce95\",\n \"name\": \"5bce1036-3d2d-444f-ad5f-2938541cce95\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:45:55.582699774Z\",\n \"endTime\": \"2026-09-22T17:09:56.885776378Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t05e9l3bms07pg766c7t42iidgiet8o still exists (deletion dispatched at 2026-09-22T16:45:57Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5973a177b4ff26eaab8a9ddb800d92fb6aa5d222f1920ce7de54e9510789aee5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412736506892288","occurred_at":"2026-09-22T15:00:31Z","failed_at":"alert","signature_id":"df11f48db0c6021e63f67dc6d870c79b45967a8dbc153562f9b9af292144b654","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T16:56:25Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j6892288-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-qxgv7\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-qxgv7 (namespace arobit) on cluster ci01-j6892288-svc has averaged 127.1% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"5bce131ddf07125511fb20c7c42b0fa12d169489d4e5931c70908bd0a1a298a8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412736506892288","occurred_at":"2026-09-22T15:00:31Z","failed_at":"e2e","signature_id":"ddf39fe4612073dbe574c94e7b39f5362ea6232c9183ce5f9846126b8b3b5e3b","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:95]: Timed out after 623.703s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000c664c0\u003e: \n failed to gather logs: failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KInternal): Error while getting token : DefaultAzureCredential: failed to acquire a token.\n Attempted credentials:\n \tClientSecretCredential: unable to resolve an endpoint: server response error:\n \t\t context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:95]: Timed out after 623.703s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000c664c0\u003e: \n failed to gather logs: failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KInternal): Error while getting token : DefaultAzureCredential: failed to acquire a token.\n Attempted credentials:\n \tClientSecretCredential: unable to resolve an endpoint: server response error:\n \t\t context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ca45167e71db14f5aeb17f2f3385a1b55e20eb87859fa05cf7e6c97421bb62be","failure_pattern":"failed to gather logs: failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KInternal): Error while getting token : DefaultAzureCredential: failed to acquire a token","search_query":"failed to gather logs: failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KInternal): Error while getting token : DefaultAzureCredential: failed to acquire a token"}},{"row_id":"b0b7097acf28b77a41fe81a8dd172327e82b84ad97cccfc787faeb0428a58ab7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412736506892288","occurred_at":"2026-09-22T15:00:31Z","failed_at":"e2e","signature_id":"4fbe849ec315aa759e6eaaa36007105f2abfc855ed8d77b0a75867781de33a52","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0016de180\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-8j45b7\" in resourcegroup=\"rg-np-version-upgrade-8j45b7-xfd6p45fzpjm\" to finish deleting: GET https://rp.j6892288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef32d17b-db28-47e9-88f8-42656f04a135\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef32d17b-db28-47e9-88f8-42656f04a135\",\n \"name\": \"ef32d17b-db28-47e9-88f8-42656f04a135\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:31:57.110038375Z\",\n \"endTime\": \"2026-09-22T16:56:06.840108207Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t05ebc137s21nadehehjbelf4umr9sv still exists (deletion dispatched at 2026-09-22T16:31:58Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0016de180\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-8j45b7\" in resourcegroup=\"rg-np-version-upgrade-8j45b7-xfd6p45fzpjm\" to finish deleting: GET https://rp.j6892288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef32d17b-db28-47e9-88f8-42656f04a135\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef32d17b-db28-47e9-88f8-42656f04a135\",\n \"name\": \"ef32d17b-db28-47e9-88f8-42656f04a135\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:31:57.110038375Z\",\n \"endTime\": \"2026-09-22T16:56:06.840108207Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t05ebc137s21nadehehjbelf4umr9sv still exists (deletion dispatched at 2026-09-22T16:31:58Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","ca45167e71db14f5aeb17f2f3385a1b55e20eb87859fa05cf7e6c97421bb62be","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412694500937728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6128,"pr_state":"open","pr_sha":"170f76835cc1a848764bf285f1e459b293c9fc67","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T15:00:21Z","started_at":"2026-09-22T15:00:21Z","completed_at":"2026-09-22T17:17:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T17:25:39.429871358Z","timing_metadata_checked_at":"2026-09-22T17:25:39.429871358Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T17:25:39.433409446Z","region_metadata_checked_at":"2026-09-22T17:25:39.433409446Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"03dddbf9f5fb25c4e18d0d968b090f775a6d25891129a454ee0ef049e0823e9e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412694500937728","occurred_at":"2026-09-22T15:00:21Z","failed_at":"e2e","signature_id":"6b4531564f88abae6232829d157ef359558a3580ebfe6e3dc7e43f0c62215fae","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0019a0120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-r6kg2bvtc2sr\" to finish deleting: GET https://rp.j0937728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/499e16c4-5cc0-4158-8dd1-44a120f33a5b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/499e16c4-5cc0-4158-8dd1-44a120f33a5b\",\n \"name\": \"499e16c4-5cc0-4158-8dd1-44a120f33a5b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:34:00.468746528Z\",\n \"endTime\": \"2026-09-22T16:58:07.479939695Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t059hkhmn5f0iq0hki8997j21ldmr5u still exists (deletion dispatched at 2026-09-22T16:34:02Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0019a0120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-r6kg2bvtc2sr\" to finish deleting: GET https://rp.j0937728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/499e16c4-5cc0-4158-8dd1-44a120f33a5b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/499e16c4-5cc0-4158-8dd1-44a120f33a5b\",\n \"name\": \"499e16c4-5cc0-4158-8dd1-44a120f33a5b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:34:00.468746528Z\",\n \"endTime\": \"2026-09-22T16:58:07.479939695Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t059hkhmn5f0iq0hki8997j21ldmr5u still exists (deletion dispatched at 2026-09-22T16:34:02Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8d6314bd6c865ce84a976357a60688fb07ac3886a0b4f6e3791da2569f821330","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412694500937728","occurred_at":"2026-09-22T15:00:21Z","failed_at":"alert","signature_id":"98654b9625183e68d8396ec4dc88e8d3aa8faa32a967d767bdc548bf01023385","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T17:03:42Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j0937728-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-nbw4j\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-nbw4j (namespace arobit) on cluster ci01-j0937728-svc has averaged 126% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"b8ddf8e56c9d18290ce4a33e437f25e810845ed590333a8adf9f346a17a22ca4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412694500937728","occurred_at":"2026-09-22T15:00:21Z","failed_at":"e2e","signature_id":"70f45997273ebdb95e6e31e8407499ba7b05f852f4b22108605ff4f1ae6eaa76","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011168b8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-69rcqx\" in resourcegroup=\"rg-np-version-upgrade-69rcqx-t7vvxb4m82c2\" to finish deleting: GET https://rp.j0937728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ff253b2-25f8-4257-bbb3-405b53cbb8c3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ff253b2-25f8-4257-bbb3-405b53cbb8c3\",\n \"name\": \"1ff253b2-25f8-4257-bbb3-405b53cbb8c3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:21:27.793968638Z\",\n \"endTime\": \"2026-09-22T16:45:27.454994019Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t059ko8q63a62o47obuopvif25t4gq7 still exists (deletion dispatched at 2026-09-22T16:21:30Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011168b8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-69rcqx\" in resourcegroup=\"rg-np-version-upgrade-69rcqx-t7vvxb4m82c2\" to finish deleting: GET https://rp.j0937728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ff253b2-25f8-4257-bbb3-405b53cbb8c3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ff253b2-25f8-4257-bbb3-405b53cbb8c3\",\n \"name\": \"1ff253b2-25f8-4257-bbb3-405b53cbb8c3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:21:27.793968638Z\",\n \"endTime\": \"2026-09-22T16:45:27.454994019Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t059ko8q63a62o47obuopvif25t4gq7 still exists (deletion dispatched at 2026-09-22T16:21:30Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102409636605857792","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6821,"pr_state":"open","pr_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T14:48:11Z","started_at":"2026-09-22T14:48:11Z","completed_at":"2026-09-22T16:51:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T17:04:03.519879957Z","timing_metadata_checked_at":"2026-09-22T17:04:03.519879957Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T17:04:03.522879247Z","region_metadata_checked_at":"2026-09-22T17:04:03.522879247Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"7e8caec606d05bc681c8ee8e418726c4e435f758b3a24aa7f81ffb073b9f6597","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102409636605857792","occurred_at":"2026-09-22T14:48:11Z","failed_at":"alert","signature_id":"dfd6f97a6359d56dafe4e7fd9a76d40da48ac489a5ba8285f1381f935bfad629","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T16:07:49Z\n Ended: 2026-09-22T16:18:48Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j5857792-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} has exceeded 250ms over the past 30 minutes (cluster ci01-j5857792-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"f4fc074380bb20ea3787759de0abf0d1712aa1bd1e5b3ccdaec669bea21b9561","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102409636605857792","occurred_at":"2026-09-22T14:48:11Z","failed_at":"alert","signature_id":"14b3fcd8667cbbc2313076de16166cf53629f131635aa37c2c3fd5d3b1e80eb6","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatency6h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T16:13:02Z\n Severity: Sev3\n Labels: alertname=\"userJourneyFrontendLatency6h30m\", cluster=\"ci01-j5857792-svc\", component=\"slo\", long_window=\"6h\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n Description: More than 6% of Frontend requests on cluster ci01-j5857792-svc exceeded 1s on both the 6h and 30m windows (6x burn of the 1% latency budget).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"addfa32c4c8681e954bead4d63f7ee51babad93b3eb924950aeb173e9712942b","failure_pattern":"alert [svc] userJourneyFrontendLatency6h30m fired","search_query":"alert [svc] userJourneyFrontendLatency6h30m fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["addfa32c4c8681e954bead4d63f7ee51babad93b3eb924950aeb173e9712942b","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7066/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102409243024953344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7066,"pr_state":"closed","pr_sha":"23179a5354aa06a57efca1f82118702c3e475ce6","final_merged_sha":"23179a5354aa06a57efca1f82118702c3e475ce6","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-22T14:46:39Z","started_at":"2026-09-22T14:46:39Z","completed_at":"2026-09-22T16:56:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T17:14:43.088330467Z","timing_metadata_checked_at":"2026-09-22T17:14:43.088330467Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T17:14:43.091878255Z","region_metadata_checked_at":"2026-09-22T17:14:43.091878255Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7079/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102409098145304576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7079,"pr_state":"closed","pr_sha":"a66c056ace7a6a9e73944032b0924658c1a5ec03","final_merged_sha":"a66c056ace7a6a9e73944032b0924658c1a5ec03","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T14:46:04Z","started_at":"2026-09-22T14:46:04Z","completed_at":"2026-09-22T16:47:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T17:04:03.815124133Z","timing_metadata_checked_at":"2026-09-22T17:04:03.815124133Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T17:04:03.819141919Z","region_metadata_checked_at":"2026-09-22T17:04:03.819141919Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"43071d1eb2590b085f3907e7d8984cc73856299a526547e1eba861bd3b5a3256","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7079/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102409098145304576","occurred_at":"2026-09-22T14:46:04Z","failed_at":"alert","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T15:47:19Z\n Ended: 2026-09-22T16:02:16Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j5304576-svc\", component=\"backend\", name=\"clusterresources\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102408702790209536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"c530dc60b8d6e25606f9f32861e6bf0caa14baec","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T14:44:25Z","started_at":"2026-09-22T14:44:25Z","completed_at":"2026-09-22T17:09:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T17:25:39.553726237Z","timing_metadata_checked_at":"2026-09-22T17:25:39.553726237Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T17:25:39.556266228Z","region_metadata_checked_at":"2026-09-22T17:25:39.556266228Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"2d89adfffa4f773c4c35a36be536208616cef29cae0c480e985c049baafc11dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102408702790209536","occurred_at":"2026-09-22T14:44:25Z","failed_at":"alert","signature_id":"83ce9eba8fd7c4927e958e70e9923c68764adb43fd3779794c7181e2062c4a07","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T16:41:53Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j0209536-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-pv857\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-pv857 (namespace arobit) on cluster ci01-j0209536-svc has averaged 126.4% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"54dd64a0dfa1d1351b0c04877a7b11c35c9a591bb129c50d56ceed8919fd84ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102408702790209536","occurred_at":"2026-09-22T14:44:25Z","failed_at":"e2e","signature_id":"be55dffe861468905d6cf5bbb952f4afc7acde7b16ebb5829460c64a53913043","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc00109e318\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-xfn7hz\" in resourcegroup=\"rg-np-skip-minor-xfn7hz-fxl2864zknf9\" to finish deleting: GET https://rp.j0209536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6ca80c75-8e81-4c60-bd28-79f67fe671f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6ca80c75-8e81-4c60-bd28-79f67fe671f4\",\n \"name\": \"6ca80c75-8e81-4c60-bd28-79f67fe671f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:23:52.521516326Z\",\n \"endTime\": \"2026-09-22T16:47:53.92980352Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t055v8ffvgh7hoqu3ka4e48gk1k5rmp still exists (deletion dispatched at 2026-09-22T16:23:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc00109e318\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-xfn7hz\" in resourcegroup=\"rg-np-skip-minor-xfn7hz-fxl2864zknf9\" to finish deleting: GET https://rp.j0209536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6ca80c75-8e81-4c60-bd28-79f67fe671f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6ca80c75-8e81-4c60-bd28-79f67fe671f4\",\n \"name\": \"6ca80c75-8e81-4c60-bd28-79f67fe671f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:23:52.521516326Z\",\n \"endTime\": \"2026-09-22T16:47:53.92980352Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t055v8ffvgh7hoqu3ka4e48gk1k5rmp still exists (deletion dispatched at 2026-09-22T16:23:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7ee308a8c4d15e9d5f3ffc8aa01347db8f249cb71aa0c5fa110ed9316e1a09bb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102408702790209536","occurred_at":"2026-09-22T14:44:25Z","failed_at":"e2e","signature_id":"93314144e1cea207b5ebb0b54cca0d2401a84bfcc54f2f1a3b60890f1aaeaa3d","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc001c8a1e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-qggbmvr2rwbm\" to finish deleting: GET https://rp.j0209536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3cc3a241-4d56-45df-ab75-ee83af76ddc0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3cc3a241-4d56-45df-ab75-ee83af76ddc0\",\n \"name\": \"3cc3a241-4d56-45df-ab75-ee83af76ddc0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:26:02.874087694Z\",\n \"endTime\": \"2026-09-22T16:50:03.912563121Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t0563ura2ok3oborcab1tonlj3tu42b still exists (deletion dispatched at 2026-09-22T16:26:04Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc001c8a1e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-qggbmvr2rwbm\" to finish deleting: GET https://rp.j0209536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3cc3a241-4d56-45df-ab75-ee83af76ddc0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3cc3a241-4d56-45df-ab75-ee83af76ddc0\",\n \"name\": \"3cc3a241-4d56-45df-ab75-ee83af76ddc0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:26:02.874087694Z\",\n \"endTime\": \"2026-09-22T16:50:03.912563121Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t0563ura2ok3oborcab1tonlj3tu42b still exists (deletion dispatched at 2026-09-22T16:26:04Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b1227cdf35cd91cca317045fce37a3562ddf3580cd256a6de25dc2f84231f82c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102408702790209536","occurred_at":"2026-09-22T14:44:25Z","failed_at":"e2e","signature_id":"f7c65b6efaad1f15cb61fc9b0be29ce6b45f3525dd78d215bd3a40750a28f9df","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.004s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc002204a80\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.004s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc002204a80\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}},{"row_id":"bb528e4f9a7fea58c433bdbac337a6cbfbfa50a74b310a5c737be101dd5fa7b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102408702790209536","occurred_at":"2026-09-22T14:44:25Z","failed_at":"e2e","signature_id":"6d3fd822dfa17e31364223c9652ee3d5734e8c3cdb06b0149b5e7838678a62af","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc001886318\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-spgcj2kh2h5v\" to finish deleting: GET https://rp.j0209536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bcde3a95-b9f7-4460-9746-02c9c95942bd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bcde3a95-b9f7-4460-9746-02c9c95942bd\",\n \"name\": \"bcde3a95-b9f7-4460-9746-02c9c95942bd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:12:37.093658582Z\",\n \"endTime\": \"2026-09-22T16:36:43.921137452Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t0566nivsb9spt22bvvibas1l9sn9op still exists (deletion dispatched at 2026-09-22T16:12:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc001886318\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-spgcj2kh2h5v\" to finish deleting: GET https://rp.j0209536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bcde3a95-b9f7-4460-9746-02c9c95942bd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bcde3a95-b9f7-4460-9746-02c9c95942bd\",\n \"name\": \"bcde3a95-b9f7-4460-9746-02c9c95942bd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:12:37.093658582Z\",\n \"endTime\": \"2026-09-22T16:36:43.921137452Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t0566nivsb9spt22bvvibas1l9sn9op still exists (deletion dispatched at 2026-09-22T16:12:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102408218671058944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7054,"pr_state":"open","pr_sha":"5126a8ab14bbdc5df1fac930f763a8387b971ef2","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T14:42:34Z","started_at":"2026-09-22T14:42:34Z","completed_at":"2026-09-22T17:04:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T17:14:42.671956141Z","timing_metadata_checked_at":"2026-09-22T17:14:42.671956141Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T17:14:42.675726528Z","region_metadata_checked_at":"2026-09-22T17:14:42.675726528Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"7836f54a0532554025d083445d6eaea31e8a1c95351454052df6c6c3e29913d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102408218671058944","occurred_at":"2026-09-22T14:42:34Z","failed_at":"e2e","signature_id":"3981163203feb69b0a9b3885b33c6c2b7c20d0dd44d64a4aed81b04a298bef52","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00176c920\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00176c920\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}},{"row_id":"a06f5afa08d5d35b59a733271ea1b1d1da85b32898604260c2aa2eb2ce76314d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102408218671058944","occurred_at":"2026-09-22T14:42:34Z","failed_at":"e2e","signature_id":"f705eec296baf9972cc07d4901b687015d1dc5a25a5effbd97e4ccc48c8a16f4","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000b962a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-cvbhkw49wbpm\" to finish deleting: GET https://rp.j1058944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f877ac3b-3a58-48db-ab63-47dc469a8550\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f877ac3b-3a58-48db-ab63-47dc469a8550\",\n \"name\": \"f877ac3b-3a58-48db-ab63-47dc469a8550\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:22:27.181566391Z\",\n \"endTime\": \"2026-09-22T16:46:34.108010104Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t053nni8oc5l9492pcvasv7p0llgkk9 still exists (deletion dispatched at 2026-09-22T16:22:28Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000b962a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-cvbhkw49wbpm\" to finish deleting: GET https://rp.j1058944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f877ac3b-3a58-48db-ab63-47dc469a8550\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f877ac3b-3a58-48db-ab63-47dc469a8550\",\n \"name\": \"f877ac3b-3a58-48db-ab63-47dc469a8550\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:22:27.181566391Z\",\n \"endTime\": \"2026-09-22T16:46:34.108010104Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t053nni8oc5l9492pcvasv7p0llgkk9 still exists (deletion dispatched at 2026-09-22T16:22:28Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102407137983467520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7020,"pr_state":"closed","pr_sha":"6cb0492059ca17ab9fe7fffef3fab0908d31e190","final_merged_sha":"6cb0492059ca17ab9fe7fffef3fab0908d31e190","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T14:38:17Z","started_at":"2026-09-22T14:38:17Z","completed_at":"2026-09-22T16:55:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T17:04:03.674094622Z","timing_metadata_checked_at":"2026-09-22T17:04:03.674094622Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T17:04:03.676782813Z","region_metadata_checked_at":"2026-09-22T17:04:03.676782813Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"1daa346b296155d79437d4339ee3e4a7fc6c760fe27353d10baba5dbfda8d703","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102407137983467520","occurred_at":"2026-09-22T14:38:17Z","failed_at":"e2e","signature_id":"39bd8a304f7e6731ee8ba3d539f47311cc9b577598f4fc5464828fc4148fc7fa","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a88078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-qdkndq\" in resourcegroup=\"rg-zstream-upgrade-4-20-mgkkt9xlp7cv\" to finish deleting: GET https://rp.j3467520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e5070f36-fffe-45fb-8025-aaa6534d21ee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e5070f36-fffe-45fb-8025-aaa6534d21ee\",\n \"name\": \"e5070f36-fffe-45fb-8025-aaa6534d21ee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:00:42.037171162Z\",\n \"endTime\": \"2026-09-22T16:24:46.401073716Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t052krp2n6diohfm8enfagipjapnjd7 still exists (deletion dispatched at 2026-09-22T16:01:06Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a88078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-qdkndq\" in resourcegroup=\"rg-zstream-upgrade-4-20-mgkkt9xlp7cv\" to finish deleting: GET https://rp.j3467520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e5070f36-fffe-45fb-8025-aaa6534d21ee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e5070f36-fffe-45fb-8025-aaa6534d21ee\",\n \"name\": \"e5070f36-fffe-45fb-8025-aaa6534d21ee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T16:00:42.037171162Z\",\n \"endTime\": \"2026-09-22T16:24:46.401073716Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t052krp2n6diohfm8enfagipjapnjd7 still exists (deletion dispatched at 2026-09-22T16:01:06Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"de539a3d5eb6d6e91ded03ee939e31e595226b08d41ddcdf120b92b243bebc4b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102407137983467520","occurred_at":"2026-09-22T14:38:17Z","failed_at":"alert","signature_id":"34f1f17119f4d188bf1ad27cd49fa9062927b1fcbdc68b9c068ecb52cad6117a","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T16:09:00Z\n Ended: 2026-09-22T16:14:59Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3467520-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3467520-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7071/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102401634003849216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7071,"pr_state":"open","pr_sha":"901730f14fbdd5486107a4de254e41a92ff32f55","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T14:16:24Z","started_at":"2026-09-22T14:16:24Z","completed_at":"2026-09-22T16:11:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T16:21:29.034405756Z","timing_metadata_checked_at":"2026-09-22T16:21:29.034405756Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T16:21:29.037694344Z","region_metadata_checked_at":"2026-09-22T16:21:29.037694344Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"2dfe9ed52618d7c58b92b7e090cc1bd12a942dc498ec8da0c5434eb88de7d025","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7071/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102401634003849216","occurred_at":"2026-09-22T14:16:24Z","failed_at":"alert","signature_id":"21991f4941f1b527a268eb9db9017a064881bad0def97631087a0d0253d8adb1","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T15:33:28Z\n Ended: 2026-09-22T15:39:28Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3849216-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3849216-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"80ea1821d06a6a26525840e9e6940cca74fc34011c41ca3ef71dc801bfd792b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7071/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102401634003849216","occurred_at":"2026-09-22T14:16:24Z","failed_at":"e2e","signature_id":"119f392f012953d3976e33cee72be8c3b09d267ec41ed4878feee3bb260cbeae","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-55rf9x\" for cluster \"cluster-candidate-5-0-55rf9x\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001027d20\u003e: \n failed to create NodePool np-55rf9x: failed waiting for nodepool=\"np-55rf9x\" for cluster \"cluster-candidate-5-0-55rf9x\" in resourcegroup=\"rg-candidate-5-0-tv9lmfcf58jh\" to finish creating: GET https://rp.j3849216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e6c15d22-5a6f-4305-86d0-76dd664e2eeb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e6c15d22-5a6f-4305-86d0-76dd664e2eeb\",\n \"name\": \"e6c15d22-5a6f-4305-86d0-76dd664e2eeb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T15:06:40.249228102Z\",\n \"endTime\": \"2026-09-22T15:25:48.538295529Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-55rf9x\" for cluster \"cluster-candidate-5-0-55rf9x\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001027d20\u003e: \n failed to create NodePool np-55rf9x: failed waiting for nodepool=\"np-55rf9x\" for cluster \"cluster-candidate-5-0-55rf9x\" in resourcegroup=\"rg-candidate-5-0-tv9lmfcf58jh\" to finish creating: GET https://rp.j3849216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e6c15d22-5a6f-4305-86d0-76dd664e2eeb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e6c15d22-5a6f-4305-86d0-76dd664e2eeb\",\n \"name\": \"e6c15d22-5a6f-4305-86d0-76dd664e2eeb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T15:06:40.249228102Z\",\n \"endTime\": \"2026-09-22T15:25:48.538295529Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102400420944023552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6662,"pr_state":"open","pr_sha":"911c458b49d2a2cd3ced7ef2fefe8affa0c9e753","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T14:11:35Z","started_at":"2026-09-22T14:11:35Z","completed_at":"2026-09-22T16:15:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T16:31:56.614439031Z","timing_metadata_checked_at":"2026-09-22T16:31:56.614439031Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T16:31:56.618408418Z","region_metadata_checked_at":"2026-09-22T16:31:56.618408418Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"6bf290d21a087cad06d02264c7ff3b6d01e6e071eb29f373d8498fa612fd0b88","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102400420944023552","occurred_at":"2026-09-22T14:11:35Z","failed_at":"e2e","signature_id":"0c4f54710868a82d6cd4ab188b8e5b92ccdc663bd7044f1d176549c7a1679554","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001ac2060\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-x9wsbz\" in resourcegroup=\"rg-zstream-upgrade-4-20-fssqcz7gkd95\" to finish deleting: GET https://rp.j4023552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8523fbca-d533-47f4-9ac4-f0f2cfb1d97e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8523fbca-d533-47f4-9ac4-f0f2cfb1d97e\",\n \"name\": \"8523fbca-d533-47f4-9ac4-f0f2cfb1d97e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T15:30:10.282474765Z\",\n \"endTime\": \"2026-09-22T15:54:15.386999815Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t04kdenf2urnkc2mp70smb8379c9eh1 still exists (deletion dispatched at 2026-09-22T15:30:14Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001ac2060\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-x9wsbz\" in resourcegroup=\"rg-zstream-upgrade-4-20-fssqcz7gkd95\" to finish deleting: GET https://rp.j4023552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8523fbca-d533-47f4-9ac4-f0f2cfb1d97e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8523fbca-d533-47f4-9ac4-f0f2cfb1d97e\",\n \"name\": \"8523fbca-d533-47f4-9ac4-f0f2cfb1d97e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T15:30:10.282474765Z\",\n \"endTime\": \"2026-09-22T15:54:15.386999815Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t04kdenf2urnkc2mp70smb8379c9eh1 still exists (deletion dispatched at 2026-09-22T15:30:14Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7072/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102399637812940800","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7072,"pr_state":"closed","pr_sha":"df1f8c96bc4134463c70614a21629c5ca8f88b97","final_merged_sha":"a1bb52523ea8e8e012393f45ec7f563124c2deed","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T14:08:23Z","started_at":"2026-09-22T14:08:23Z","completed_at":"2026-09-22T16:26:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T16:42:56.510452528Z","timing_metadata_checked_at":"2026-09-22T16:42:56.510452528Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T16:42:56.513721516Z","region_metadata_checked_at":"2026-09-22T16:42:56.513721516Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"256dd803c94b0b5a293630260a24058232aade297b860baf7858c6beddc232e1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7072/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102399637812940800","occurred_at":"2026-09-22T14:08:23Z","failed_at":"e2e","signature_id":"501591f861d7f3032330c71c8a8110b35bd8c46439dfa6416ccd3c8fa3fc9d55","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00136a120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-z2xhl2smhl95\" to finish deleting: GET https://rp.j2940800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/020b2ad2-90c1-42cf-954f-9c42af86b89a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/020b2ad2-90c1-42cf-954f-9c42af86b89a\",\n \"name\": \"020b2ad2-90c1-42cf-954f-9c42af86b89a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T15:45:07.411300012Z\",\n \"endTime\": \"2026-09-22T16:09:12.675827628Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t04j35581secir5jt8vd82f1nps1e1h still exists (deletion dispatched at 2026-09-22T15:45:09Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00136a120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-z2xhl2smhl95\" to finish deleting: GET https://rp.j2940800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/020b2ad2-90c1-42cf-954f-9c42af86b89a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/020b2ad2-90c1-42cf-954f-9c42af86b89a\",\n \"name\": \"020b2ad2-90c1-42cf-954f-9c42af86b89a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T15:45:07.411300012Z\",\n \"endTime\": \"2026-09-22T16:09:12.675827628Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t04j35581secir5jt8vd82f1nps1e1h still exists (deletion dispatched at 2026-09-22T15:45:09Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"84e35d0aaeb57ef3bf9037613537d029d31b890299b2b4ca525e7f085c5d7fa8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7072/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102399637812940800","occurred_at":"2026-09-22T14:08:23Z","failed_at":"alert","signature_id":"0b0c62094340da20a3f28022dd2ae1bbdc1bcfdc0b4bcf968fa111ba233899d0","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T16:03:28Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j2940800-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-px2fd\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-px2fd (namespace arobit) on cluster ci01-j2940800-svc has averaged 126.3% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102397479768035328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6922,"pr_state":"open","pr_sha":"87e40a07142a1457ed3f83374cc5d085c87d8f29","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T13:59:53Z","started_at":"2026-09-22T13:59:53Z","completed_at":"2026-09-22T16:14:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T16:31:57.018883097Z","timing_metadata_checked_at":"2026-09-22T16:31:57.018883097Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T16:31:57.023487882Z","region_metadata_checked_at":"2026-09-22T16:31:57.023487882Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"579f775867077a08ac6030dc285cbab97d9842483a069a9797ab7b4cfa7ecc03","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102397479768035328","occurred_at":"2026-09-22T13:59:53Z","failed_at":"e2e","signature_id":"7e9d178e6c594517bad8d5499ee753b7577341b05fba4f102298178f01197cde","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00113c168\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-kgt4dp\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-4927vw69rw8m\" to finish deleting: GET https://rp.j8035328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1e0cb39-53d3-4858-ad0a-0f3a35aac6ba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1e0cb39-53d3-4858-ad0a-0f3a35aac6ba\",\n \"name\": \"e1e0cb39-53d3-4858-ad0a-0f3a35aac6ba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T15:13:19.013850014Z\",\n \"endTime\": \"2026-09-22T15:37:24.201740062Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t04dltg78ddmiii1fbhn53kmu5p1uov still exists (deletion dispatched at 2026-09-22T15:13:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00113c168\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-kgt4dp\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-4927vw69rw8m\" to finish deleting: GET https://rp.j8035328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1e0cb39-53d3-4858-ad0a-0f3a35aac6ba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1e0cb39-53d3-4858-ad0a-0f3a35aac6ba\",\n \"name\": \"e1e0cb39-53d3-4858-ad0a-0f3a35aac6ba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T15:13:19.013850014Z\",\n \"endTime\": \"2026-09-22T15:37:24.201740062Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t04dltg78ddmiii1fbhn53kmu5p1uov still exists (deletion dispatched at 2026-09-22T15:13:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"90b99f9f39f9ba9cb6007fd9c712fa082bcb5a173dfd0954cb155078733cd2b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102397479768035328","occurred_at":"2026-09-22T13:59:53Z","failed_at":"alert","signature_id":"a0369e4b4747ddf65f74013118f1b2acd9e1cd85149284da363bc09e58ad009b","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T15:54:51Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j8035328-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-xf56d\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-xf56d (namespace arobit) on cluster ci01-j8035328-svc has averaged 124% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"add97dcef1966a834a7e66fdb2c88020452e9a10c470d4c490020cfd135945f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102397479768035328","occurred_at":"2026-09-22T13:59:53Z","failed_at":"e2e","signature_id":"bd2d83ef948cc00c872335719fff75d3ab902eb5b2a91c73605693d9a4d11d71","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a120a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-ff54k9w27kdm\" to finish deleting: GET https://rp.j8035328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/71c3d3ae-fcf9-46a8-a5ab-5481a9f5283e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/71c3d3ae-fcf9-46a8-a5ab-5481a9f5283e\",\n \"name\": \"71c3d3ae-fcf9-46a8-a5ab-5481a9f5283e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T15:31:35.352639854Z\",\n \"endTime\": \"2026-09-22T15:55:44.858066336Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t04dmjhorlcc8rbknfj93atobs76t0i still exists (deletion dispatched at 2026-09-22T15:31:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a120a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-ff54k9w27kdm\" to finish deleting: GET https://rp.j8035328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/71c3d3ae-fcf9-46a8-a5ab-5481a9f5283e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/71c3d3ae-fcf9-46a8-a5ab-5481a9f5283e\",\n \"name\": \"71c3d3ae-fcf9-46a8-a5ab-5481a9f5283e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T15:31:35.352639854Z\",\n \"endTime\": \"2026-09-22T15:55:44.858066336Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t04dmjhorlcc8rbknfj93atobs76t0i still exists (deletion dispatched at 2026-09-22T15:31:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7077/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102391598355582976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7077,"pr_state":"closed","pr_sha":"c12349d46ed2c22a9bf226166bdf8f91e648e32e","final_merged_sha":"53e6edac2bb7b25fcabf41daf657c529a89c7f7e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T13:36:14Z","started_at":"2026-09-22T13:36:14Z","completed_at":"2026-09-22T15:35:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T15:50:29.667244392Z","timing_metadata_checked_at":"2026-09-22T15:50:29.667244392Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T15:50:29.670061083Z","region_metadata_checked_at":"2026-09-22T15:50:29.670061083Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3b22c5f7f25ce4f833d2c0e5e813d949e4471cebe6d70bf7fac38506f47991f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7077/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102391598355582976","occurred_at":"2026-09-22T13:36:14Z","failed_at":"e2e","signature_id":"312f92f173f393934c2af6c3b15128820da22eb3a757e3a667a9dabf4eb5433b","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n \u003c*errors.joinError | 0xc001514030\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-ctpxj.apps.aro.v4m3f6y4q7z0t3b.chi6.j5582976.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-ctpxj.apps.aro.v4m3f6y4q7z0t3b.chi6.j5582976.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-ctpxj.apps.aro.v4m3f6y4q7z0t3b.chi6.j5582976.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n \u003c*errors.joinError | 0xc001514030\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-ctpxj.apps.aro.v4m3f6y4q7z0t3b.chi6.j5582976.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-ctpxj.apps.aro.v4m3f6y4q7z0t3b.chi6.j5582976.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-ctpxj.apps.aro.v4m3f6y4q7z0t3b.chi6.j5582976.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a87df562eaa8a0bfda8c14840c36e7197e35a3b92f22e4f536b1e3d2ed289ff","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-ctpxj.apps.aro.v4m3f6y4q7z0t3b.chi6.j5582976.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-ctpxj.apps.aro.v4m3f6y4q7z0t3b.chi6.j5582976.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-ctpxj.apps.aro.v4m3f6y4q7z0t3b.chi6.j5582976.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7a87df562eaa8a0bfda8c14840c36e7197e35a3b92f22e4f536b1e3d2ed289ff"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102388132350529536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7001,"pr_state":"open","pr_sha":"604bf185122d7880b7c0266bf60627acba568296","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T13:22:45Z","started_at":"2026-09-22T13:22:45Z","completed_at":"2026-09-22T15:32:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T15:50:29.497043951Z","timing_metadata_checked_at":"2026-09-22T15:50:29.497043951Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T15:50:29.500586239Z","region_metadata_checked_at":"2026-09-22T15:50:29.500586239Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"03ab440fd03719203150b77c5e57327b5975985b4a6a5af4ce614316715c073d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102388132350529536","occurred_at":"2026-09-22T13:22:45Z","failed_at":"e2e","signature_id":"ce439a01f3dd078882fa3337ada81f946640cfb68bece27f3dc970d7a2c530b0","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012680c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-kzfns6p8hnnz\" to finish deleting: GET https://rp.j0529536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/357dbfd6-1ecc-49c9-8451-726c0525914c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/357dbfd6-1ecc-49c9-8451-726c0525914c\",\n \"name\": \"357dbfd6-1ecc-49c9-8451-726c0525914c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T14:51:53.865556884Z\",\n \"endTime\": \"2026-09-22T15:16:02.876997439Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t03qlefes7qqu9ndhbba63al41fsd54 still exists (deletion dispatched at 2026-09-22T14:51:55Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012680c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-kzfns6p8hnnz\" to finish deleting: GET https://rp.j0529536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/357dbfd6-1ecc-49c9-8451-726c0525914c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/357dbfd6-1ecc-49c9-8451-726c0525914c\",\n \"name\": \"357dbfd6-1ecc-49c9-8451-726c0525914c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T14:51:53.865556884Z\",\n \"endTime\": \"2026-09-22T15:16:02.876997439Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t03qlefes7qqu9ndhbba63al41fsd54 still exists (deletion dispatched at 2026-09-22T14:51:55Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a058a05879c495d03b80e44262afbeb9669da4f010e650b990900b250d5b979f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102388132350529536","occurred_at":"2026-09-22T13:22:45Z","failed_at":"alert","signature_id":"69a4d01934f71c8d4cc5f95af06ab52f13a41aae602819428092cff5854cdee4","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T14:34:27Z\n Ended: 2026-09-22T14:47:29Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j0529536-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j0529536-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"ce3a438c1140838207e49310091aa333300061b11c50c0a973740db8ffcabc39","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102388132350529536","occurred_at":"2026-09-22T13:22:45Z","failed_at":"alert","signature_id":"42ed4dd43bfd0f010d040aeeeb49ff87ef925441b0a50891a0c9f45eef3a169e","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T15:19:33Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j0529536-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-zt76m\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-zt76m (namespace arobit) on cluster ci01-j0529536-svc has averaged 127% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102385490215833600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6688,"pr_state":"open","pr_sha":"f8180cd6044fd44b58d22fb50e2a7d9429087dfe","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T13:12:15Z","started_at":"2026-09-22T13:12:15Z","completed_at":"2026-09-22T15:29:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T15:39:34.781361575Z","timing_metadata_checked_at":"2026-09-22T15:39:34.781361575Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T15:39:34.784810364Z","region_metadata_checked_at":"2026-09-22T15:39:34.784810364Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"01b4c57890f5caae8dfac4de8a2d69e05adf4b1acdb6519170039569e5dca04e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102385490215833600","occurred_at":"2026-09-22T13:12:15Z","failed_at":"e2e","signature_id":"3aef474821203f0513acbdb2bd4f218706dfce0520096977bbd87f680e92cc40","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011701e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-z58j7fdlqz95\" to finish deleting: GET https://rp.j5833600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/684e8ee0-2c18-411e-acc4-32c72b510d88\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/684e8ee0-2c18-411e-acc4-32c72b510d88\",\n \"name\": \"684e8ee0-2c18-411e-acc4-32c72b510d88\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T14:47:35.770349631Z\",\n \"endTime\": \"2026-09-22T15:11:39.446675519Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t03nuv85l4gu6j6jj010pj9d9bc1a7p still exists (deletion dispatched at 2026-09-22T14:47:36Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011701e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-z58j7fdlqz95\" to finish deleting: GET https://rp.j5833600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/684e8ee0-2c18-411e-acc4-32c72b510d88\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/684e8ee0-2c18-411e-acc4-32c72b510d88\",\n \"name\": \"684e8ee0-2c18-411e-acc4-32c72b510d88\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T14:47:35.770349631Z\",\n \"endTime\": \"2026-09-22T15:11:39.446675519Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t03nuv85l4gu6j6jj010pj9d9bc1a7p still exists (deletion dispatched at 2026-09-22T14:47:36Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6538591c482994497e88ebc76e282eda6f6ef5e39fd2fcbc84ab10f1f958938a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102385490215833600","occurred_at":"2026-09-22T13:12:15Z","failed_at":"e2e","signature_id":"bc47151889f90300b321741d89b94df457be37cf4936f4c5a0af9551d05801fd","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:251]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n \u003c*errors.joinError | 0xc0025d8000\u003e: \n context deadline exceeded\n failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:251]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n \u003c*errors.joinError | 0xc0025d8000\u003e: \n context deadline exceeded\n failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"759896226e154b302a585253120538810010d916bb7d065f2634acceb56a3416","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102385490215833600","occurred_at":"2026-09-22T13:12:15Z","failed_at":"alert","signature_id":"c7981f65a6291a8ebee62a5b85aa3f6972edca569a1b1203b0c8479137b02024","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T15:07:54Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j5833600-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-cn6gk\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-cn6gk (namespace arobit) on cluster ci01-j5833600-svc has averaged 125.3% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"f965190eb614f76569e8ab2a4c0193dc8abe06ac70e4c37acc3113e370f0e3b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102385490215833600","occurred_at":"2026-09-22T13:12:15Z","failed_at":"alert","signature_id":"52816a71ef2a03b49578fdbb7a916c3c123b4226a4221c05983979cb58643e9c","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T14:26:28Z\n Ended: 2026-09-22T14:42:27Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j5833600-svc\", component=\"backend\", name=\"datadump\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue datadump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7067/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102382907266961408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7067,"pr_state":"closed","pr_sha":"8ae5d545296859679e6c957bce4ca9e5af6e376c","final_merged_sha":"8ae5d545296859679e6c957bce4ca9e5af6e376c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T13:02:00Z","started_at":"2026-09-22T13:02:00Z","completed_at":"2026-09-22T15:18:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T15:29:30.577389706Z","timing_metadata_checked_at":"2026-09-22T15:29:30.577389706Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T15:29:30.582210291Z","region_metadata_checked_at":"2026-09-22T15:29:30.582210291Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"4783203bbb718e8f80f1dd69b4d26ba49504a6abc0f43108d56c0923ad202fb4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7067/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102382907266961408","occurred_at":"2026-09-22T13:02:00Z","failed_at":"e2e","signature_id":"27062e67a1d88224902adb713a26696bb6c66d8e52b518684911402a1e0836d3","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000b0e6c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-psj6bqhkk247\" to finish deleting: GET https://rp.j6961408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b1507794-ddcc-4272-8150-61c8e2db2951\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b1507794-ddcc-4272-8150-61c8e2db2951\",\n \"name\": \"b1507794-ddcc-4272-8150-61c8e2db2951\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T14:37:48.593192113Z\",\n \"endTime\": \"2026-09-22T15:01:49.089274756Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t03iul7c5s89v3g3tp3eoc5uotn7s1q still exists (deletion dispatched at 2026-09-22T14:37:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000b0e6c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-psj6bqhkk247\" to finish deleting: GET https://rp.j6961408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b1507794-ddcc-4272-8150-61c8e2db2951\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b1507794-ddcc-4272-8150-61c8e2db2951\",\n \"name\": \"b1507794-ddcc-4272-8150-61c8e2db2951\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T14:37:48.593192113Z\",\n \"endTime\": \"2026-09-22T15:01:49.089274756Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t03iul7c5s89v3g3tp3eoc5uotn7s1q still exists (deletion dispatched at 2026-09-22T14:37:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7066/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102382882726088704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7066,"pr_state":"closed","pr_sha":"eca364775bafae8fcd8399b7d0db91dc6fb25c6d","final_merged_sha":"23179a5354aa06a57efca1f82118702c3e475ce6","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T13:01:50Z","started_at":"2026-09-22T13:01:50Z","completed_at":"2026-09-22T13:53:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T14:05:51.718537331Z","timing_metadata_checked_at":"2026-09-22T14:05:51.718537331Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T14:05:51.723131717Z","region_metadata_checked_at":"2026-09-22T14:05:51.723131717Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"889dbe85f6b752c2f74980101023290ef031a878f01ba90195d387ce6194d837","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7066/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102382882726088704","occurred_at":"2026-09-22T13:01:50Z","failed_at":"provision","signature_id":"6bac7c9167143bb4a1dfabfc4a6d81083bef0f715dcabad843cd680ef59999d4","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/prometheus","test_suite":"templatize-pipeline","failure_text":"time=2026-09-22T13:40:24.327Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-09-22T13:40:25.067Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus values=\"map[environment:arohcpci01 kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:9e63919b9dde3a44483a565c55744abf9eab54314586f01a631a42b1104d5f2a] metricLabelsAllowlist:[nodes=[node.kubernetes.io/instance-type,kubernetes.azure.com/agentpool]]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:01038e7de14b78d702d2849c3aad72fd25903c4765af63cf16aa3398f5d5f2dd]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:be4e1b8bf4b8cc868cfa993818ed94e2536adc15713fc64942ce6a47d1bbabb9] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:b840d65d98ba99f4d62329db721714b36ec23d7a0c8721d1fdb4fc839f97cf76]] thanosImage:map[registry:arohcpsvcdev.azurecr.io repository:thanos/thanos sha:b567818fe608067eb0f1d7c2c4fe361e7ad83c8a256234c97685f1d0bf670cc8 tag:v0.41.0] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[serviceAccount:map[managedIdentity:d7ccda51-6374-4c36-847f-8c68f35d13dc]] prometheusSpec:map[externalLabels:map[cluster:ci01-j6088704-svc environment:ci01 region:westus3] hcpRemoteWriteUrl:NONE image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:92fa8f5e91a0e02e0107c346f6c66657738dbf394946e52c208150fd498554bd] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-ci01-j6088704-svc-ybo2.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-7b09a45e774144ada99a9dc87a23c02f/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 replicas:2 resources:map[limits:map[memory:NONE] requests:map[cpu:NONE memory:NONE]] shards:1 version: zoneCount:3]]\"\ntime=2026-09-22T13:40:25.067Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-09-22T13:40:25.331Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus namespace=prometheus\ntime=2026-09-22T13:40:25.636Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus dryRun=false\ntime=2026-09-22T13:40:25.636Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-09-22T13:40:25.815Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-09-22T13:46:35.923Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"failed pre-install: resource Job/prometheus/prometheus-admission-create not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-09-22T13:46:35.923Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed pre-install: resource Job/prometheus/prometheus-admission-create not ready. status: Failed, message: Job Failed. failed: 1/1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0984e2155380bc7af2966b58c7068460e7d241fdb82676269ebd409413b3d478","failure_pattern":"failed to roll out Helm release: failed pre-install: resource Job/prometheus/prometheus-admission-create not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed pre-install: resource Job/prometheus/prometheus-admission-create not ready. status: Failed, message: Job Failed. failed: 1/1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0984e2155380bc7af2966b58c7068460e7d241fdb82676269ebd409413b3d478"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102382852271247360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6503,"pr_state":"closed","pr_sha":"6c5cfd09b9bbf1585b1cd37220da16677f327452","final_merged_sha":"a0bf660bfdf32222374a82406e0907cae4b53869","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T13:01:46Z","started_at":"2026-09-22T13:01:46Z","completed_at":"2026-09-22T15:05:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T15:19:22.149338701Z","timing_metadata_checked_at":"2026-09-22T15:19:22.149338701Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T15:19:22.152680689Z","region_metadata_checked_at":"2026-09-22T15:19:22.152680689Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"2dc8ce56cd68a0fd5590e87e8b61458f642a22ccca8297521d126fb8d28dd4bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102382852271247360","occurred_at":"2026-09-22T13:01:46Z","failed_at":"e2e","signature_id":"be110f491d5014ebfa7f9314a49d9edae412a9f9b47e864700f474e4f5c5ebce","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00199c120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-gvrrfd7mjt5j\" to finish deleting: GET https://rp.j1247360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5c52fcaf-7a1b-409f-8d01-0b1f6fe3e2eb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5c52fcaf-7a1b-409f-8d01-0b1f6fe3e2eb\",\n \"name\": \"5c52fcaf-7a1b-409f-8d01-0b1f6fe3e2eb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T14:22:32.067338295Z\",\n \"endTime\": \"2026-09-22T14:46:39.174252031Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t03iv79memg0f0b55eshd59cvgqg187 still exists (deletion dispatched at 2026-09-22T14:22:33Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00199c120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-gvrrfd7mjt5j\" to finish deleting: GET https://rp.j1247360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5c52fcaf-7a1b-409f-8d01-0b1f6fe3e2eb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5c52fcaf-7a1b-409f-8d01-0b1f6fe3e2eb\",\n \"name\": \"5c52fcaf-7a1b-409f-8d01-0b1f6fe3e2eb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T14:22:32.067338295Z\",\n \"endTime\": \"2026-09-22T14:46:39.174252031Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t03iv79memg0f0b55eshd59cvgqg187 still exists (deletion dispatched at 2026-09-22T14:22:33Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9c1caa220cda236ddeaa39c9a31634f9dea86aa2ca0cf55853498b94b00b6625","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102382852271247360","occurred_at":"2026-09-22T13:01:46Z","failed_at":"alert","signature_id":"5a293d3681a6cb16a3acf3c0acdf1d5978054ad425ca5fb3f977d8e2311584eb","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T13:56:14Z\n Ended: 2026-09-22T14:30:13Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1247360-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1247360-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6889,"pr_state":"open","pr_sha":"01ae0b3a1f3882e2293627daaa78faf9da8f05f3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T12:46:33Z","started_at":"2026-09-22T12:46:33Z","completed_at":"2026-09-22T14:56:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T15:08:34.402615649Z","timing_metadata_checked_at":"2026-09-22T15:08:34.402615649Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T15:08:34.406592835Z","region_metadata_checked_at":"2026-09-22T15:08:34.406592835Z"},"failed_at":["alert","e2e"],"failed_test_count":63,"occurrences":[{"row_id":"0a5b79f60f2d245e0d75c5904ede5134cd92818fe71802ca0c7fd3511756124f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"c42e87688be4015e90e55364d94e2cfdf70fc7be2f1df40b5b9a613da178bd5b","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa07e0\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-9vb62w25cp44\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8709ce20-9ece-4f8c-adc6-a03474e00ed8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8709ce20-9ece-4f8c-adc6-a03474e00ed8\",\n \"name\": \"8709ce20-9ece-4f8c-adc6-a03474e00ed8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:48.411427372Z\",\n \"endTime\": \"2026-09-22T14:01:54.964813171Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa07e0\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-9vb62w25cp44\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8709ce20-9ece-4f8c-adc6-a03474e00ed8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8709ce20-9ece-4f8c-adc6-a03474e00ed8\",\n \"name\": \"8709ce20-9ece-4f8c-adc6-a03474e00ed8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:48.411427372Z\",\n \"endTime\": \"2026-09-22T14:01:54.964813171Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"179c952f1321699c79bc7a9d3dee7090a70566c96294ac633a6b31b18c4e994e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"e3628da3891e1ec49619dde7547596c92e9e1c1cfe73fca8833c1206090506a8","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"1dea568132054bdd28699f1c9833531dbf356526d61ce387b4cfbbee34c8b4e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"64d81e706a219ab0e795d214a529e53451bd79f2cae0bb71190437d2d9a85d3f","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-tpxqzt\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000548ca0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-tpxqzt: failed waiting for cluster=\"cp-ystream-upgrade-4-22-tpxqzt\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-kzmrc9msf5wb\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c339ee69-a414-453c-a869-0268b31874e6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c339ee69-a414-453c-a869-0268b31874e6\",\n \"name\": \"c339ee69-a414-453c-a869-0268b31874e6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:20.2901057Z\",\n \"endTime\": \"2026-09-22T14:01:24.891016711Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-tpxqzt\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000548ca0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-tpxqzt: failed waiting for cluster=\"cp-ystream-upgrade-4-22-tpxqzt\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-kzmrc9msf5wb\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c339ee69-a414-453c-a869-0268b31874e6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c339ee69-a414-453c-a869-0268b31874e6\",\n \"name\": \"c339ee69-a414-453c-a869-0268b31874e6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:20.2901057Z\",\n \"endTime\": \"2026-09-22T14:01:24.891016711Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"26c9fe10f9a55de0761276a87731e3e55aad9ff07ef627984af7371c7a8b532c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"d5dbf4b895cbcceda926d5e609a4c7ccd9fd21955e94c103a607056e3c975392","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001034880\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-kh9mclv7h744\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/957231f3-0c5e-474f-967b-9b836b1bcff8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/957231f3-0c5e-474f-967b-9b836b1bcff8\",\n \"name\": \"957231f3-0c5e-474f-967b-9b836b1bcff8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:34:03.567742056Z\",\n \"endTime\": \"2026-09-22T14:03:04.97548831Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001034880\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-kh9mclv7h744\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/957231f3-0c5e-474f-967b-9b836b1bcff8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/957231f3-0c5e-474f-967b-9b836b1bcff8\",\n \"name\": \"957231f3-0c5e-474f-967b-9b836b1bcff8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:34:03.567742056Z\",\n \"endTime\": \"2026-09-22T14:03:04.97548831Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2a65a62943ad9dd2cdc87c827e8c7c2ff4d4fcddbd4d033c59b2f20c19f419a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"67667a59b4a551350f8d79402114e1c840bbf22a863b684a641fd345472b2e4c","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f266c0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-k2p6znpk7xx7\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef41379b-6f3e-4285-8db4-e555e4ff61f6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef41379b-6f3e-4285-8db4-e555e4ff61f6\",\n \"name\": \"ef41379b-6f3e-4285-8db4-e555e4ff61f6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:27.09771924Z\",\n \"endTime\": \"2026-09-22T14:01:34.874183273Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f266c0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-k2p6znpk7xx7\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef41379b-6f3e-4285-8db4-e555e4ff61f6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef41379b-6f3e-4285-8db4-e555e4ff61f6\",\n \"name\": \"ef41379b-6f3e-4285-8db4-e555e4ff61f6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:27.09771924Z\",\n \"endTime\": \"2026-09-22T14:01:34.874183273Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2ebea4f545fb4488c7754a6808f2b37a59d7ab1362a3761dfa5a7fcd89d33923","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"41f4c527fb82523b18c53454eebee84e13213799c50b84ac6f5104114806a006","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001812570\u003e: \n GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3eb8ee2-faee-44d1-b236-05c68194f7e8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3eb8ee2-faee-44d1-b236-05c68194f7e8\",\n \"name\": \"c3eb8ee2-faee-44d1-b236-05c68194f7e8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:46.407843603Z\",\n \"endTime\": \"2026-09-22T14:01:54.958239729Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001812570\u003e: \n GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3eb8ee2-faee-44d1-b236-05c68194f7e8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3eb8ee2-faee-44d1-b236-05c68194f7e8\",\n \"name\": \"c3eb8ee2-faee-44d1-b236-05c68194f7e8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:46.407843603Z\",\n \"endTime\": \"2026-09-22T14:01:54.958239729Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"33e4b5c9bedd68527272f73a42c8d1e801b82a28e704951844ce21c2151a2c99","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"125d7b2a1e64c2c36598f3d8585ea4489c6478937c146dfc9a85d370146b92da","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a60ba0\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-862l4sm6zmr9\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c11a3871-fc26-4435-a797-b908d4806163\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c11a3871-fc26-4435-a797-b908d4806163\",\n \"name\": \"c11a3871-fc26-4435-a797-b908d4806163\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:31:51.078510413Z\",\n \"endTime\": \"2026-09-22T14:00:54.733017321Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a60ba0\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-862l4sm6zmr9\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c11a3871-fc26-4435-a797-b908d4806163\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c11a3871-fc26-4435-a797-b908d4806163\",\n \"name\": \"c11a3871-fc26-4435-a797-b908d4806163\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:31:51.078510413Z\",\n \"endTime\": \"2026-09-22T14:00:54.733017321Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"368d5f12461e9bb5643d4a24a84f4450eb12a1c8bf4b6691dfc69814a4a4aa65","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"75c907116ecaab8397397d390ad70c1823faa982df325a591793d7d6259ebb46","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ed48e0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-fbhlhhg249qs\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/42374bce-7c8e-4634-9787-3fba14ef004b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/42374bce-7c8e-4634-9787-3fba14ef004b\",\n \"name\": \"42374bce-7c8e-4634-9787-3fba14ef004b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T14:08:04.562790307Z\",\n \"endTime\": \"2026-09-22T14:37:08.529968518Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ed48e0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-fbhlhhg249qs\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/42374bce-7c8e-4634-9787-3fba14ef004b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/42374bce-7c8e-4634-9787-3fba14ef004b\",\n \"name\": \"42374bce-7c8e-4634-9787-3fba14ef004b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T14:08:04.562790307Z\",\n \"endTime\": \"2026-09-22T14:37:08.529968518Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"37655cb5611e8185856456709fe2864d6be770221b0ccea80f8c2767b9b13b23","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"1077ec918010bde6d0e63b9e0cb832071c18f4ca2227a1a6e2444c0f54217012","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-kbjcpq\" with version 4.21.33 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f728e0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-kbjcpq: failed waiting for cluster=\"cluster-zstream-4-21-kbjcpq\" in resourcegroup=\"rg-zstream-upgrade-4-21-ftfjp89m25kk\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d11e0bc-f5d8-4182-a6e1-387ef6e21ad9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d11e0bc-f5d8-4182-a6e1-387ef6e21ad9\",\n \"name\": \"3d11e0bc-f5d8-4182-a6e1-387ef6e21ad9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:22.061338875Z\",\n \"endTime\": \"2026-09-22T14:01:24.909726517Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-kbjcpq\" with version 4.21.33 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f728e0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-kbjcpq: failed waiting for cluster=\"cluster-zstream-4-21-kbjcpq\" in resourcegroup=\"rg-zstream-upgrade-4-21-ftfjp89m25kk\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d11e0bc-f5d8-4182-a6e1-387ef6e21ad9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d11e0bc-f5d8-4182-a6e1-387ef6e21ad9\",\n \"name\": \"3d11e0bc-f5d8-4182-a6e1-387ef6e21ad9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:22.061338875Z\",\n \"endTime\": \"2026-09-22T14:01:24.909726517Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3a9677cb10360b1bd3cfc560a9ec82cb3a57ed0ff4bd195b4e5a06a75c12df3e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"7867693174b52991f98e5b358683d9b1a30f9bef5582588f00189e01aa4a23b6","test_name":"Fleet should have scheduling data for ready management clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:162]: Timed out after 900.001s.\nscheduling data was not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:156 with:\nstamp 2 scheduling requested CPU must be positive\nExpected\n \u003cbool\u003e: ...\nto be false\nfail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:162]: Timed out after 900.001s.\nscheduling data was not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:156 with:\nstamp 2 scheduling requested CPU must be positive\nExpected\n \u003cbool\u003e: ...\nto be false","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0cd69af8904472f73d329003c8c5db9d0e892958b3af0d73d1a1dee951ccff6d","failure_pattern":"scheduling requested CPU must be positive","search_query":"stamp 2 scheduling requested CPU must be positive"}},{"row_id":"3cea14e3091958a4011cedf272d09f6879e424fff18a5281e7ffd1b535115687","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"e5241bfbb33f7be045964d3491b282166f16778f8ac1f32bdf2e605f8ad176d7","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bb8c80\u003e: \n failed to create HCP cluster np-dg-minor-8jkpl9: failed waiting for cluster=\"np-dg-minor-8jkpl9\" in resourcegroup=\"rg-np-dg-minor-8jkpl9-97vgtw4swh8v\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1dc855f6-8924-4660-af02-dc66ad12ebdd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1dc855f6-8924-4660-af02-dc66ad12ebdd\",\n \"name\": \"1dc855f6-8924-4660-af02-dc66ad12ebdd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:27.973621119Z\",\n \"endTime\": \"2026-09-22T14:01:34.946432956Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bb8c80\u003e: \n failed to create HCP cluster np-dg-minor-8jkpl9: failed waiting for cluster=\"np-dg-minor-8jkpl9\" in resourcegroup=\"rg-np-dg-minor-8jkpl9-97vgtw4swh8v\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1dc855f6-8924-4660-af02-dc66ad12ebdd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1dc855f6-8924-4660-af02-dc66ad12ebdd\",\n \"name\": \"1dc855f6-8924-4660-af02-dc66ad12ebdd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:27.973621119Z\",\n \"endTime\": \"2026-09-22T14:01:34.946432956Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"40a0f975f9687bbeb3884e1959f23de79bcb5ced4adb3df9639862c280a7d0d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"3f191fc29b9245b9c91f145004f0e87445cff99d866e034bacd2dc82342ffd6c","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-xbw9gf\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f22840\u003e: \n failed to create HCP cluster np-noedge-xbw9gf: failed waiting for cluster=\"np-noedge-xbw9gf\" in resourcegroup=\"rg-np-noedge-xbw9gf-2g5w2t9xwwpv\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3013569-9710-4d56-9127-660e8d0e8b4e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3013569-9710-4d56-9127-660e8d0e8b4e\",\n \"name\": \"e3013569-9710-4d56-9127-660e8d0e8b4e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:47.584413552Z\",\n \"endTime\": \"2026-09-22T14:01:54.952468852Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-xbw9gf\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f22840\u003e: \n failed to create HCP cluster np-noedge-xbw9gf: failed waiting for cluster=\"np-noedge-xbw9gf\" in resourcegroup=\"rg-np-noedge-xbw9gf-2g5w2t9xwwpv\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3013569-9710-4d56-9127-660e8d0e8b4e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3013569-9710-4d56-9127-660e8d0e8b4e\",\n \"name\": \"e3013569-9710-4d56-9127-660e8d0e8b4e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:47.584413552Z\",\n \"endTime\": \"2026-09-22T14:01:54.952468852Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"48e9865befd62b96319402a53ed0f01e59d4dedc97d7498ca3e0fe4a5c2b6b89","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"49da713cdfdaa83760481f71b81afd1e546eed486287452f59b4494f14665db5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"945dba197673b6f485c758b73f9756c8272cd1cff7d79dfa0c33b87db2f91428","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-5jzdhm\" with version 5.0.0-rc.2 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010a2600\u003e: \n failed to create HCP cluster cluster-zstream-5-0-5jzdhm: failed waiting for cluster=\"cluster-zstream-5-0-5jzdhm\" in resourcegroup=\"rg-zstream-upgrade-5-0-kt8kw5kcc6ns\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21ebda2b-c622-4b56-b92a-871dda0eed85\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21ebda2b-c622-4b56-b92a-871dda0eed85\",\n \"name\": \"21ebda2b-c622-4b56-b92a-871dda0eed85\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:01.665343445Z\",\n \"endTime\": \"2026-09-22T14:01:04.637676887Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-5jzdhm\" with version 5.0.0-rc.2 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010a2600\u003e: \n failed to create HCP cluster cluster-zstream-5-0-5jzdhm: failed waiting for cluster=\"cluster-zstream-5-0-5jzdhm\" in resourcegroup=\"rg-zstream-upgrade-5-0-kt8kw5kcc6ns\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21ebda2b-c622-4b56-b92a-871dda0eed85\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21ebda2b-c622-4b56-b92a-871dda0eed85\",\n \"name\": \"21ebda2b-c622-4b56-b92a-871dda0eed85\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:01.665343445Z\",\n \"endTime\": \"2026-09-22T14:01:04.637676887Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4a4d158a66637976d70741c842f5a185485236b4262860e463ba2d6d5eeaf442","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"62c72b9790eeaa622371cd4fcce3f1a55c9e869af1565ae1854130331bf7ee17","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d87020\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-64pg9485vnxx\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d2fbe0d2-aad1-4d01-bb33-d19f8d8b8c62\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d2fbe0d2-aad1-4d01-bb33-d19f8d8b8c62\",\n \"name\": \"d2fbe0d2-aad1-4d01-bb33-d19f8d8b8c62\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:28.884296464Z\",\n \"endTime\": \"2026-09-22T14:01:34.90419391Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d87020\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-64pg9485vnxx\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d2fbe0d2-aad1-4d01-bb33-d19f8d8b8c62\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d2fbe0d2-aad1-4d01-bb33-d19f8d8b8c62\",\n \"name\": \"d2fbe0d2-aad1-4d01-bb33-d19f8d8b8c62\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:28.884296464Z\",\n \"endTime\": \"2026-09-22T14:01:34.90419391Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4af752258546b61fa6cec48c19dcdb55ab32ab590cdcbd0440bd43837013ed21","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"7b61254d3b2425439673bf81a0dda06b0179ac653363cac4b7966dce8a437e5f","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:83]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000de6840\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-htnbsdgmvcr4\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb215c1d-0a1a-47ba-a73d-78e5f47e89af\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb215c1d-0a1a-47ba-a73d-78e5f47e89af\",\n \"name\": \"cb215c1d-0a1a-47ba-a73d-78e5f47e89af\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:27.21502087Z\",\n \"endTime\": \"2026-09-22T14:01:34.964236451Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:83]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000de6840\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-htnbsdgmvcr4\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb215c1d-0a1a-47ba-a73d-78e5f47e89af\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb215c1d-0a1a-47ba-a73d-78e5f47e89af\",\n \"name\": \"cb215c1d-0a1a-47ba-a73d-78e5f47e89af\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:27.21502087Z\",\n \"endTime\": \"2026-09-22T14:01:34.964236451Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4d191f228a8c2eb62a0d984fc44a6c78ac5af23232406446cae4761f8b33fa99","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"c412aaa9e3b24cac733542e008d37a90317f70a3ea34642e1c14406364c1ee06","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-mtqdfc with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a98480\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-mtqdfc: failed waiting for cluster=\"np-version-upgrade-cluster-mtqdfc\" in resourcegroup=\"rg-np-version-upgrade-mtqdfc-bth59zhjnz6n\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f65ee424-6cf6-4989-a7d1-891071500c60\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f65ee424-6cf6-4989-a7d1-891071500c60\",\n \"name\": \"f65ee424-6cf6-4989-a7d1-891071500c60\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:28.003057597Z\",\n \"endTime\": \"2026-09-22T14:01:34.644902606Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-mtqdfc with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a98480\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-mtqdfc: failed waiting for cluster=\"np-version-upgrade-cluster-mtqdfc\" in resourcegroup=\"rg-np-version-upgrade-mtqdfc-bth59zhjnz6n\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f65ee424-6cf6-4989-a7d1-891071500c60\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f65ee424-6cf6-4989-a7d1-891071500c60\",\n \"name\": \"f65ee424-6cf6-4989-a7d1-891071500c60\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:28.003057597Z\",\n \"endTime\": \"2026-09-22T14:01:34.644902606Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4fc58d1228531e674d11065728aceecd94d00da6c5f160b215e29bcfabf13f95","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"alert","signature_id":"4b82640645f0248147b9da24bd012fa2cd836d5c931b44904d7cc22a4314f873","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors1h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T14:07:46Z\n Severity: Sev3\n Labels: alertname=\"userJourneyClusterProvisionErrors1h5m\", cluster=\"ci01-j7129984-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7129984/providers/microsoft.monitor/accounts/services-j7129984\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"cluster-provision-errors\"\n Description: More than 72% of completed cluster create (install) operations on ci01-j7129984-svc failed over the last hour with at least 3 failures, a 14.4x burn of the 95% SLO budget.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9a8854d173302655314e0bdcbc8411f9e276ffe0aadf4e25f0e0bad8f8dcb51d","failure_pattern":"alert [svc] userJourneyClusterProvisionErrors1h5m fired","search_query":"alert [svc] userJourneyClusterProvisionErrors1h5m fired"}},{"row_id":"5133abf1959cfcc0e76bcb007b48654d7c519300b52dd368a5f738f48d5fc496","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"e08c3b837f2d8a64eba9a61bf8290a329e4e4689019bc4c9c513e0eb9c9e8227","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001606840\u003e: \n failed to create HCP cluster np-downgrade-m7w6pj: failed waiting for cluster=\"np-downgrade-m7w6pj\" in resourcegroup=\"rg-np-downgrade-m7w6pj-b4q6lx8fd9gs\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f56cada-f16e-4e2f-ac20-e517ffdaf734\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f56cada-f16e-4e2f-ac20-e517ffdaf734\",\n \"name\": \"8f56cada-f16e-4e2f-ac20-e517ffdaf734\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:49.748056499Z\",\n \"endTime\": \"2026-09-22T14:01:54.973155468Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001606840\u003e: \n failed to create HCP cluster np-downgrade-m7w6pj: failed waiting for cluster=\"np-downgrade-m7w6pj\" in resourcegroup=\"rg-np-downgrade-m7w6pj-b4q6lx8fd9gs\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f56cada-f16e-4e2f-ac20-e517ffdaf734\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f56cada-f16e-4e2f-ac20-e517ffdaf734\",\n \"name\": \"8f56cada-f16e-4e2f-ac20-e517ffdaf734\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:49.748056499Z\",\n \"endTime\": \"2026-09-22T14:01:54.973155468Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"535a4f12d496247b430528f09b486918ec7effba24c7e085a3835c5d569e77cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"05bb22ed0e241c1167007404c8d4e3b91da97b4772f7ce448b2d4b48caf1c2a6","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d58a20\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-g7q8vnwmjvvk\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ea0270f1-32ca-4b4d-a131-126d53d2800a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ea0270f1-32ca-4b4d-a131-126d53d2800a\",\n \"name\": \"ea0270f1-32ca-4b4d-a131-126d53d2800a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T14:06:37.339670093Z\",\n \"endTime\": \"2026-09-22T14:35:38.595765601Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d58a20\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-g7q8vnwmjvvk\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ea0270f1-32ca-4b4d-a131-126d53d2800a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ea0270f1-32ca-4b4d-a131-126d53d2800a\",\n \"name\": \"ea0270f1-32ca-4b4d-a131-126d53d2800a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T14:06:37.339670093Z\",\n \"endTime\": \"2026-09-22T14:35:38.595765601Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"53627dcdcb4edb5bfd4dff766791cb191836ba9f79be62d72ac61d5fa3446422","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"736b4003f514984286a44fd82ba86089bf1764ca185c1772eeac867d501348c1","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-m6rtch with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a6e4a0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-m6rtch: failed waiting for cluster=\"np-version-upgrade-cluster-m6rtch\" in resourcegroup=\"rg-np-version-upgrade-m6rtch-kp76f866ck8n\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ea5323bc-7d7a-4cd2-9069-577a587fdb12\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ea5323bc-7d7a-4cd2-9069-577a587fdb12\",\n \"name\": \"ea5323bc-7d7a-4cd2-9069-577a587fdb12\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:57.760074103Z\",\n \"endTime\": \"2026-09-22T14:02:04.634127705Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-m6rtch with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a6e4a0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-m6rtch: failed waiting for cluster=\"np-version-upgrade-cluster-m6rtch\" in resourcegroup=\"rg-np-version-upgrade-m6rtch-kp76f866ck8n\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ea5323bc-7d7a-4cd2-9069-577a587fdb12\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ea5323bc-7d7a-4cd2-9069-577a587fdb12\",\n \"name\": \"ea5323bc-7d7a-4cd2-9069-577a587fdb12\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:57.760074103Z\",\n \"endTime\": \"2026-09-22T14:02:04.634127705Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"54a79e9c002d300c5a08bb7e18e24f8036d9f74684dc8c0562ff9a072024ffb1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"26bf64aa34ca1abd16a8823ecf0cbcbcaee37cb43a5cfe37931577af830a986f","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c90ba0\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-9pjbfw8psrpq\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb5b03af-09d4-44d3-b02e-ce58d25b6b3b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb5b03af-09d4-44d3-b02e-ce58d25b6b3b\",\n \"name\": \"cb5b03af-09d4-44d3-b02e-ce58d25b6b3b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:01.886561261Z\",\n \"endTime\": \"2026-09-22T14:01:04.632033687Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c90ba0\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-9pjbfw8psrpq\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb5b03af-09d4-44d3-b02e-ce58d25b6b3b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cb5b03af-09d4-44d3-b02e-ce58d25b6b3b\",\n \"name\": \"cb5b03af-09d4-44d3-b02e-ce58d25b6b3b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:01.886561261Z\",\n \"endTime\": \"2026-09-22T14:01:04.632033687Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"573d40dcf71fcb77c05b6aaaf154685d69dbcc2b17bded9489ba03d5eece518c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"91d76ee6d298962789879c47d4545a4200ff5cbefc6f9bc106201b6ae385dd8b","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-th49nv with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000def320\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-th49nv: failed waiting for cluster=\"np-version-upgrade-cluster-th49nv\" in resourcegroup=\"rg-np-version-upgrade-th49nv-j4l6x9jgvj9t\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80f13221-7309-42c2-8783-0028aa3526c6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80f13221-7309-42c2-8783-0028aa3526c6\",\n \"name\": \"80f13221-7309-42c2-8783-0028aa3526c6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:48.394279998Z\",\n \"endTime\": \"2026-09-22T14:01:54.565780738Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-th49nv with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000def320\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-th49nv: failed waiting for cluster=\"np-version-upgrade-cluster-th49nv\" in resourcegroup=\"rg-np-version-upgrade-th49nv-j4l6x9jgvj9t\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80f13221-7309-42c2-8783-0028aa3526c6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80f13221-7309-42c2-8783-0028aa3526c6\",\n \"name\": \"80f13221-7309-42c2-8783-0028aa3526c6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:48.394279998Z\",\n \"endTime\": \"2026-09-22T14:01:54.565780738Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5ca12071f94804a523a3207f6e8b48de0f518543d116f3a8d8b8031fbe525e6e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"4c9febb5685c00820373c671b942a0ad6d4af8fb290a59e20e05e9220aa6393c","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000958220\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-92jh9cwk45qj\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/20b25f8b-196a-49c8-a2f2-8e0d2717aa00\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/20b25f8b-196a-49c8-a2f2-8e0d2717aa00\",\n \"name\": \"20b25f8b-196a-49c8-a2f2-8e0d2717aa00\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:33:15.551368137Z\",\n \"endTime\": \"2026-09-22T14:02:24.66646054Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000958220\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-92jh9cwk45qj\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/20b25f8b-196a-49c8-a2f2-8e0d2717aa00\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/20b25f8b-196a-49c8-a2f2-8e0d2717aa00\",\n \"name\": \"20b25f8b-196a-49c8-a2f2-8e0d2717aa00\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:33:15.551368137Z\",\n \"endTime\": \"2026-09-22T14:02:24.66646054Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6046468aa27b5392e58ffb6fa3c3b4b0d704de9367712d9de1ec25bb0c6b66be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"22aedc52ca836cdf7d514d3fba3d71461df6de6eb162c1dcba0ac4544196e286","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-rmbgws\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001702980\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-rmbgws: failed waiting for cluster=\"cp-ystream-upgrade-4-21-rmbgws\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-z5xs8zlcnjkk\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f6e84d1d-17fd-4447-b870-5ab61c77bdf1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f6e84d1d-17fd-4447-b870-5ab61c77bdf1\",\n \"name\": \"f6e84d1d-17fd-4447-b870-5ab61c77bdf1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:19.46291759Z\",\n \"endTime\": \"2026-09-22T14:01:24.769248375Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-rmbgws\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001702980\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-rmbgws: failed waiting for cluster=\"cp-ystream-upgrade-4-21-rmbgws\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-z5xs8zlcnjkk\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f6e84d1d-17fd-4447-b870-5ab61c77bdf1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f6e84d1d-17fd-4447-b870-5ab61c77bdf1\",\n \"name\": \"f6e84d1d-17fd-4447-b870-5ab61c77bdf1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:19.46291759Z\",\n \"endTime\": \"2026-09-22T14:01:24.769248375Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"605c382d08cd47b08a44c695bccc61e641b28e8c3cb143b4817b4369557ec5c1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"27132487c8d28776a72cc37eed74057cd528324d345f115013efb32ee9878f10","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce8920\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-4th9vj6whvx7\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/30794bc9-050b-4975-a1bb-7ea957f163fb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/30794bc9-050b-4975-a1bb-7ea957f163fb\",\n \"name\": \"30794bc9-050b-4975-a1bb-7ea957f163fb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:20.953844818Z\",\n \"endTime\": \"2026-09-22T14:01:24.858958675Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce8920\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-4th9vj6whvx7\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/30794bc9-050b-4975-a1bb-7ea957f163fb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/30794bc9-050b-4975-a1bb-7ea957f163fb\",\n \"name\": \"30794bc9-050b-4975-a1bb-7ea957f163fb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:20.953844818Z\",\n \"endTime\": \"2026-09-22T14:01:24.858958675Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"616171e607ea4e042084369d78652436599415429b2b168e501f5ac230fae6f1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"97a6d366d3affe44798b8e6531d0e15d83c56eb0785ae18b4d157c51025dce96","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00148a820\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-ql6nc95drl8l\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7d253a4-df32-4561-8861-2e0882591562\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7d253a4-df32-4561-8861-2e0882591562\",\n \"name\": \"f7d253a4-df32-4561-8861-2e0882591562\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:48.802075858Z\",\n \"endTime\": \"2026-09-22T14:01:54.971192416Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00148a820\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-ql6nc95drl8l\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7d253a4-df32-4561-8861-2e0882591562\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7d253a4-df32-4561-8861-2e0882591562\",\n \"name\": \"f7d253a4-df32-4561-8861-2e0882591562\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:48.802075858Z\",\n \"endTime\": \"2026-09-22T14:01:54.971192416Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6ded136a6652e3912f033885d41bba6078f9e14917f20ceceebad98ca66cb505","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"b63c6b57c4e89b577bd5c804453853880b014ecdcd220586c0a9bbfec37a1675","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000a123f0\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-57bkrhmpnrrx\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-57bkrhmpnrrx, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000a123f0\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-57bkrhmpnrrx\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-57bkrhmpnrrx, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-57bkrhmpnrrx\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-57bkrhmpnrrx, error: context deadline exceeded"}},{"row_id":"70508b32c7a7bfc97efc1744393e16ba6bdc48223580972029fa48061ac6fe93","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"db2c996d9ab000ec5408777eab57f880c1c9caaa8695141c93132b0072cbbeba","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--n5xwmlbc98xb/cluster-hs-pre-b9ffdp should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00108eaa0\u003e: \n failed to create HCP cluster cluster-hs-pre-b9ffdp: failed waiting for cluster=\"cluster-hs-pre-b9ffdp\" in resourcegroup=\"rg-hs-pre--n5xwmlbc98xb\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1c451f0-78db-458f-9923-7716b40c6c90\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1c451f0-78db-458f-9923-7716b40c6c90\",\n \"name\": \"e1c451f0-78db-458f-9923-7716b40c6c90\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:22.063711473Z\",\n \"endTime\": \"2026-09-22T14:01:24.664579405Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--n5xwmlbc98xb/cluster-hs-pre-b9ffdp should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00108eaa0\u003e: \n failed to create HCP cluster cluster-hs-pre-b9ffdp: failed waiting for cluster=\"cluster-hs-pre-b9ffdp\" in resourcegroup=\"rg-hs-pre--n5xwmlbc98xb\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1c451f0-78db-458f-9923-7716b40c6c90\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1c451f0-78db-458f-9923-7716b40c6c90\",\n \"name\": \"e1c451f0-78db-458f-9923-7716b40c6c90\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:22.063711473Z\",\n \"endTime\": \"2026-09-22T14:01:24.664579405Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"74b8a81060ce0f17d3f19f97d2c127738d1b554916451bf4e20c41ac9e40a48b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"be58d2e7a0303b1f567347f14f904f74f000e733efb20ae68e901f4e2a588a20","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00108a1c0\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-2tnmgccpx9km\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bfe5c5f3-e516-408f-8ab3-b1c6d692820b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bfe5c5f3-e516-408f-8ab3-b1c6d692820b\",\n \"name\": \"bfe5c5f3-e516-408f-8ab3-b1c6d692820b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:22.208739639Z\",\n \"endTime\": \"2026-09-22T14:01:24.939813833Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00108a1c0\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-2tnmgccpx9km\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bfe5c5f3-e516-408f-8ab3-b1c6d692820b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bfe5c5f3-e516-408f-8ab3-b1c6d692820b\",\n \"name\": \"bfe5c5f3-e516-408f-8ab3-b1c6d692820b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:22.208739639Z\",\n \"endTime\": \"2026-09-22T14:01:24.939813833Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"784a2dacdfee5b769883e358f2dd6b209fb0ad8e9835faa352c25f3bc6a09c8e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"b03bb3905da0a0d584082f97b972b91c8e4e3cddf5cf41ebfe6b43d7c5c61e53","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a08be0\u003e: \n failed to create HCP cluster ea-listcqmtxr: failed waiting for cluster=\"ea-listcqmtxr\" in resourcegroup=\"ea-list-542l4vff7xkq\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef207e33-9e75-44d1-9a69-d8b4514144d0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef207e33-9e75-44d1-9a69-d8b4514144d0\",\n \"name\": \"ef207e33-9e75-44d1-9a69-d8b4514144d0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:25.585548746Z\",\n \"endTime\": \"2026-09-22T14:01:34.938634584Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a08be0\u003e: \n failed to create HCP cluster ea-listcqmtxr: failed waiting for cluster=\"ea-listcqmtxr\" in resourcegroup=\"ea-list-542l4vff7xkq\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef207e33-9e75-44d1-9a69-d8b4514144d0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef207e33-9e75-44d1-9a69-d8b4514144d0\",\n \"name\": \"ef207e33-9e75-44d1-9a69-d8b4514144d0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:25.585548746Z\",\n \"endTime\": \"2026-09-22T14:01:34.938634584Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"78d73e918c2f3040b42d2ce74432fe289370e12ccb83a8d14122270678103e40","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"eb1eec2d9d98e544944ca5331858c8445c1124a22d49b02bcbddc76788729d0a","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001250920\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-6jbj2h648xbl\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c35ce14b-2ef3-4651-bdad-b2703112d6f1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c35ce14b-2ef3-4651-bdad-b2703112d6f1\",\n \"name\": \"c35ce14b-2ef3-4651-bdad-b2703112d6f1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:34:52.214220949Z\",\n \"endTime\": \"2026-09-22T14:04:02.008733809Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001250920\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-6jbj2h648xbl\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c35ce14b-2ef3-4651-bdad-b2703112d6f1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c35ce14b-2ef3-4651-bdad-b2703112d6f1\",\n \"name\": \"c35ce14b-2ef3-4651-bdad-b2703112d6f1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:34:52.214220949Z\",\n \"endTime\": \"2026-09-22T14:04:02.008733809Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7adb75076da6539aa01ee6c767d70bed26850f91cee6ee4a0a7bd57afc082d1b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"4ea5bdfd0c0fe13c83e36ea93cb4d6b05660b725346e80b5ea45e3584fde7ecb","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-sxjznjq8jpsn/cluster-candidate-4-22-wmljjz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001122120\u003e: \n failed to create HCP cluster cluster-candidate-4-22-wmljjz: failed waiting for cluster=\"cluster-candidate-4-22-wmljjz\" in resourcegroup=\"rg-candidate-4-22-sxjznjq8jpsn\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7addf21d-a67b-4865-b5b3-afec4261e5ef\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7addf21d-a67b-4865-b5b3-afec4261e5ef\",\n \"name\": \"7addf21d-a67b-4865-b5b3-afec4261e5ef\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:21.967597365Z\",\n \"endTime\": \"2026-09-22T14:01:24.832286014Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-sxjznjq8jpsn/cluster-candidate-4-22-wmljjz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001122120\u003e: \n failed to create HCP cluster cluster-candidate-4-22-wmljjz: failed waiting for cluster=\"cluster-candidate-4-22-wmljjz\" in resourcegroup=\"rg-candidate-4-22-sxjznjq8jpsn\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7addf21d-a67b-4865-b5b3-afec4261e5ef\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7addf21d-a67b-4865-b5b3-afec4261e5ef\",\n \"name\": \"7addf21d-a67b-4865-b5b3-afec4261e5ef\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:21.967597365Z\",\n \"endTime\": \"2026-09-22T14:01:24.832286014Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7f446affe50d977c8ce3310b31becb74b92a2c5bf089c1e78dbb1fad0b087b7e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"f39fe90d35e76033ec3a3f6c1f03513b571ac19cd4414444db52b5e759592585","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009f8660\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-wnpc7fdg4jjt\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95fff587-94f2-447e-bc0d-a6f5af7dd01f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95fff587-94f2-447e-bc0d-a6f5af7dd01f\",\n \"name\": \"95fff587-94f2-447e-bc0d-a6f5af7dd01f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:57.870353747Z\",\n \"endTime\": \"2026-09-22T14:02:04.600472123Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009f8660\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-wnpc7fdg4jjt\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95fff587-94f2-447e-bc0d-a6f5af7dd01f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95fff587-94f2-447e-bc0d-a6f5af7dd01f\",\n \"name\": \"95fff587-94f2-447e-bc0d-a6f5af7dd01f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:57.870353747Z\",\n \"endTime\": \"2026-09-22T14:02:04.600472123Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8296b2910698151c88cf4869ea2194ff27a9fc24f3942c540b83b685cc373dd8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"8008fba97339baa5071274c98aa0e725ca29436c7f8afc131f7de5f1e67df6e0","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c5ecc0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-q8wnpjlbcjd2\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/77d0c1ba-ef9b-44da-bac1-09a4d6bc9c79\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/77d0c1ba-ef9b-44da-bac1-09a4d6bc9c79\",\n \"name\": \"77d0c1ba-ef9b-44da-bac1-09a4d6bc9c79\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:20.803439313Z\",\n \"endTime\": \"2026-09-22T14:01:24.833690805Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c5ecc0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-q8wnpjlbcjd2\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/77d0c1ba-ef9b-44da-bac1-09a4d6bc9c79\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/77d0c1ba-ef9b-44da-bac1-09a4d6bc9c79\",\n \"name\": \"77d0c1ba-ef9b-44da-bac1-09a4d6bc9c79\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:20.803439313Z\",\n \"endTime\": \"2026-09-22T14:01:24.833690805Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"82dbc36ebdbef07d3d51cd61a6b0201f0252a884d60358ef2d1ec21aebebbc13","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"9766ef3fa70c1671f3802b255d196eba0c78634f0e552de05e0ed3a4c8949254","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:88]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b77d80\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-79cwkrjwz9cs\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ba01c81e-c5a2-45f4-bd07-7eb9f1de61bf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ba01c81e-c5a2-45f4-bd07-7eb9f1de61bf\",\n \"name\": \"ba01c81e-c5a2-45f4-bd07-7eb9f1de61bf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:22.58553866Z\",\n \"endTime\": \"2026-09-22T14:01:24.895864789Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:88]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b77d80\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-79cwkrjwz9cs\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ba01c81e-c5a2-45f4-bd07-7eb9f1de61bf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ba01c81e-c5a2-45f4-bd07-7eb9f1de61bf\",\n \"name\": \"ba01c81e-c5a2-45f4-bd07-7eb9f1de61bf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:22.58553866Z\",\n \"endTime\": \"2026-09-22T14:01:24.895864789Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"83a11c984b3cd4b2e5cda1b19124138f36e00ae6c22877cddd32204a6323cd52","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"25e161c3cabcd69470473f37e02d3636693f8520b93e53149858575c5f7ca818","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-7sr4g9qqtq49/cluster-candidate-4-21-n4s875 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f7a4e0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-n4s875: failed waiting for cluster=\"cluster-candidate-4-21-n4s875\" in resourcegroup=\"rg-candidate-4-21-7sr4g9qqtq49\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/41bada58-020b-4e77-89a8-255bde10db94\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/41bada58-020b-4e77-89a8-255bde10db94\",\n \"name\": \"41bada58-020b-4e77-89a8-255bde10db94\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:31:50.90986267Z\",\n \"endTime\": \"2026-09-22T14:00:54.82147705Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-7sr4g9qqtq49/cluster-candidate-4-21-n4s875 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f7a4e0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-n4s875: failed waiting for cluster=\"cluster-candidate-4-21-n4s875\" in resourcegroup=\"rg-candidate-4-21-7sr4g9qqtq49\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/41bada58-020b-4e77-89a8-255bde10db94\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/41bada58-020b-4e77-89a8-255bde10db94\",\n \"name\": \"41bada58-020b-4e77-89a8-255bde10db94\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:31:50.90986267Z\",\n \"endTime\": \"2026-09-22T14:00:54.82147705Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"88ef3d610ca9f3ed10aab8d3f8a54ecf96b473a7c8d65442107b073dca1c7c18","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"1b7078ed292312c9ffe8816194aaa85647b1c2e4bfcf09a97c4dadbd137c675b","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00102e520\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-ccvxf6h9dzgs\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ee709e4b-d97c-4079-933f-9b1ba906ba99\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ee709e4b-d97c-4079-933f-9b1ba906ba99\",\n \"name\": \"ee709e4b-d97c-4079-933f-9b1ba906ba99\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:46.874292726Z\",\n \"endTime\": \"2026-09-22T14:01:54.873323023Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00102e520\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-ccvxf6h9dzgs\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ee709e4b-d97c-4079-933f-9b1ba906ba99\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ee709e4b-d97c-4079-933f-9b1ba906ba99\",\n \"name\": \"ee709e4b-d97c-4079-933f-9b1ba906ba99\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:46.874292726Z\",\n \"endTime\": \"2026-09-22T14:01:54.873323023Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8c02efa6f7950c2a70429c1cfd90b340e2409bf98c34668e352948861cea48e6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"alert","signature_id":"498fd6b520c3adb9da74466053b7266c919395d71a055e8926fce3e947cf3a23","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors6h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T14:32:47Z\n Severity: Sev3\n Labels: alertname=\"userJourneyClusterProvisionErrors6h30m\", cluster=\"ci01-j7129984-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7129984/providers/microsoft.monitor/accounts/services-j7129984\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n Description: More than 30% of completed cluster create (install) operations on ci01-j7129984-svc failed over the last 6 hours (at least 5 completions), a 6x burn of the 95% SLO budget.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d11e817743c9b7b223c4f7499ea10417bad7afb08d112eac38001716535808b0","failure_pattern":"alert [svc] userJourneyClusterProvisionErrors6h30m fired","search_query":"alert [svc] userJourneyClusterProvisionErrors6h30m fired"}},{"row_id":"8cd99f851e186e1fe4528755606dd8bbe66f04c9dbee14c9bf83c59ca84e321c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"35442fafa4814db9dadf16a96800b231c95dba26154f748a821779c1c450c5ca","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fea300\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-s6f9ps8n479f\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd8729f7-dafd-47d9-b4c6-5199ca0e7104\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd8729f7-dafd-47d9-b4c6-5199ca0e7104\",\n \"name\": \"cd8729f7-dafd-47d9-b4c6-5199ca0e7104\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:47.611769362Z\",\n \"endTime\": \"2026-09-22T14:01:54.819460445Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fea300\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-s6f9ps8n479f\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd8729f7-dafd-47d9-b4c6-5199ca0e7104\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd8729f7-dafd-47d9-b4c6-5199ca0e7104\",\n \"name\": \"cd8729f7-dafd-47d9-b4c6-5199ca0e7104\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:47.611769362Z\",\n \"endTime\": \"2026-09-22T14:01:54.819460445Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8d79952e9a622305581b78f831b843ba3adb55136425a0f68e050c62f612812a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"bc965970f4ca24037e7e7ce859c4604c1f5f11358722e75e0aec3f0a8e4c60fa","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001022b80\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-fmbd5h9tdkzn\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6a185f7e-c196-4cca-9649-3a45bca25626\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6a185f7e-c196-4cca-9649-3a45bca25626\",\n \"name\": \"6a185f7e-c196-4cca-9649-3a45bca25626\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:01.53469242Z\",\n \"endTime\": \"2026-09-22T14:01:04.986880418Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001022b80\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-fmbd5h9tdkzn\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6a185f7e-c196-4cca-9649-3a45bca25626\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6a185f7e-c196-4cca-9649-3a45bca25626\",\n \"name\": \"6a185f7e-c196-4cca-9649-3a45bca25626\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:01.53469242Z\",\n \"endTime\": \"2026-09-22T14:01:04.986880418Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8ea0931fe11a8cfb387d7589f9d7975819e1de0429925c82d68dff85d4815ff2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"3cb932c0e4265483c436c655e1b30c54e08b393cc31eecbbc2f5bc67afd2613b","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00072e440\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-lmh8xtrqwsz8\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e94fb788-b80f-4ab9-98a4-a704b55c9089\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e94fb788-b80f-4ab9-98a4-a704b55c9089\",\n \"name\": \"e94fb788-b80f-4ab9-98a4-a704b55c9089\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:35.011447868Z\",\n \"endTime\": \"2026-09-22T14:01:44.849684341Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00072e440\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-lmh8xtrqwsz8\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e94fb788-b80f-4ab9-98a4-a704b55c9089\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e94fb788-b80f-4ab9-98a4-a704b55c9089\",\n \"name\": \"e94fb788-b80f-4ab9-98a4-a704b55c9089\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:35.011447868Z\",\n \"endTime\": \"2026-09-22T14:01:44.849684341Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a1d2965b93cb23591b7df22eac91c062a9d764dc22aebaafb8498d18947ac7e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"f22aceb749f662a2e174bb580556604682086f921403ec06374941a12db33605","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-2cjqpskxzwwh/cluster-candidate-4-20-gdcb55 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00117c120\u003e: \n failed to create HCP cluster cluster-candidate-4-20-gdcb55: failed waiting for cluster=\"cluster-candidate-4-20-gdcb55\" in resourcegroup=\"rg-candidate-4-20-2cjqpskxzwwh\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/403eb990-4d95-45bc-b1d8-871f6a3f66f3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/403eb990-4d95-45bc-b1d8-871f6a3f66f3\",\n \"name\": \"403eb990-4d95-45bc-b1d8-871f6a3f66f3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:22.904006297Z\",\n \"endTime\": \"2026-09-22T14:01:24.628150051Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-2cjqpskxzwwh/cluster-candidate-4-20-gdcb55 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00117c120\u003e: \n failed to create HCP cluster cluster-candidate-4-20-gdcb55: failed waiting for cluster=\"cluster-candidate-4-20-gdcb55\" in resourcegroup=\"rg-candidate-4-20-2cjqpskxzwwh\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/403eb990-4d95-45bc-b1d8-871f6a3f66f3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/403eb990-4d95-45bc-b1d8-871f6a3f66f3\",\n \"name\": \"403eb990-4d95-45bc-b1d8-871f6a3f66f3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:22.904006297Z\",\n \"endTime\": \"2026-09-22T14:01:24.628150051Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a5d80ba28165e76ac48ca79fe259ef52b600314a63bc0ab93df6362830f63055","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"7f08de17fd820b423433af5f14d7253b852e4fb89158f413aa90ffd6c09047c8","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce8a00\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-7b2cjlpnvnvc\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/48483c11-8284-4a72-b0e8-808c25d5a875\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/48483c11-8284-4a72-b0e8-808c25d5a875\",\n \"name\": \"48483c11-8284-4a72-b0e8-808c25d5a875\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:27.071336756Z\",\n \"endTime\": \"2026-09-22T14:01:34.588882658Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce8a00\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-7b2cjlpnvnvc\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/48483c11-8284-4a72-b0e8-808c25d5a875\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/48483c11-8284-4a72-b0e8-808c25d5a875\",\n \"name\": \"48483c11-8284-4a72-b0e8-808c25d5a875\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:27.071336756Z\",\n \"endTime\": \"2026-09-22T14:01:34.588882658Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"abefe4db1f6edbd4614f99191863c53008f7e66364c0742c4056a6f75541507c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"0f3709ad519f7e7b8e8abc75a8dee44b2ab12631351203d1466b075bd3f2b05c","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010c0b80\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-qwx5tf2dtpk2\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e31a2de-3b83-48d4-a41a-2116a2e676ce\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e31a2de-3b83-48d4-a41a-2116a2e676ce\",\n \"name\": \"5e31a2de-3b83-48d4-a41a-2116a2e676ce\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:01.635132061Z\",\n \"endTime\": \"2026-09-22T14:01:04.903065926Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010c0b80\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-qwx5tf2dtpk2\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e31a2de-3b83-48d4-a41a-2116a2e676ce\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e31a2de-3b83-48d4-a41a-2116a2e676ce\",\n \"name\": \"5e31a2de-3b83-48d4-a41a-2116a2e676ce\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:01.635132061Z\",\n \"endTime\": \"2026-09-22T14:01:04.903065926Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ac890a27ccb8fb19827ec8fe350d6c1ca9f903a8a199b5aabffb6c9f1e4851fc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"ccd547377a6a9f24e00759530e1b5da0ca0add6799d2896e9d0d48f1085974f1","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001656760\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-m9xd8z5rbsjl\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/276a3d3f-6f71-48de-a173-d167c5d98fe6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/276a3d3f-6f71-48de-a173-d167c5d98fe6\",\n \"name\": \"276a3d3f-6f71-48de-a173-d167c5d98fe6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:02.228959747Z\",\n \"endTime\": \"2026-09-22T14:01:04.56945549Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001656760\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-m9xd8z5rbsjl\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/276a3d3f-6f71-48de-a173-d167c5d98fe6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/276a3d3f-6f71-48de-a173-d167c5d98fe6\",\n \"name\": \"276a3d3f-6f71-48de-a173-d167c5d98fe6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:02.228959747Z\",\n \"endTime\": \"2026-09-22T14:01:04.56945549Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ad96284eaf71f14cd8ec493673f7438747dc8270e71291015754b75a7859c4c2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"5c546180824a9822a969b1cdd8466c19f6ac94f9341bc046d0d0d71b44d017a2","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00039ed80\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-ln96hxc65p2r\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0a399061-f866-4bd2-911c-6ed39af5b54d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0a399061-f866-4bd2-911c-6ed39af5b54d\",\n \"name\": \"0a399061-f866-4bd2-911c-6ed39af5b54d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:17.617030211Z\",\n \"endTime\": \"2026-09-22T14:01:24.711758901Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00039ed80\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-ln96hxc65p2r\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0a399061-f866-4bd2-911c-6ed39af5b54d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0a399061-f866-4bd2-911c-6ed39af5b54d\",\n \"name\": \"0a399061-f866-4bd2-911c-6ed39af5b54d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:17.617030211Z\",\n \"endTime\": \"2026-09-22T14:01:24.711758901Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ae572073887eb37e558640be7d17809c87e1e1f1dbbc7e43812cea29bec30ca4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"4e6005083fa58df0062477155be492344adead24679f90800563c50dcb10e3e1","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eac380\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-gg78jpjwb6l4\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27a48f2f-dae8-4b01-9ddd-35f85f532068\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27a48f2f-dae8-4b01-9ddd-35f85f532068\",\n \"name\": \"27a48f2f-dae8-4b01-9ddd-35f85f532068\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:38.178380757Z\",\n \"endTime\": \"2026-09-22T14:01:44.75918192Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eac380\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-gg78jpjwb6l4\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27a48f2f-dae8-4b01-9ddd-35f85f532068\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27a48f2f-dae8-4b01-9ddd-35f85f532068\",\n \"name\": \"27a48f2f-dae8-4b01-9ddd-35f85f532068\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:38.178380757Z\",\n \"endTime\": \"2026-09-22T14:01:44.75918192Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"af551e26c19c33f88c57a9b5e221506006bf069650d4a770d87f3372e7ea8bba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"d6a3a01776dbf01fafbc37aa5433cf753bfef466e178054f8f54f9b28b7343ba","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e4a200\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-cxwvpl2jz4z4\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7233baf0-2045-40c1-a77e-4bcc134eee10\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7233baf0-2045-40c1-a77e-4bcc134eee10\",\n \"name\": \"7233baf0-2045-40c1-a77e-4bcc134eee10\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:22.052170362Z\",\n \"endTime\": \"2026-09-22T14:01:24.90059412Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e4a200\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-cxwvpl2jz4z4\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7233baf0-2045-40c1-a77e-4bcc134eee10\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7233baf0-2045-40c1-a77e-4bcc134eee10\",\n \"name\": \"7233baf0-2045-40c1-a77e-4bcc134eee10\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:22.052170362Z\",\n \"endTime\": \"2026-09-22T14:01:24.90059412Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b64ab154b42a69b921186df05ff41179ef191f5bd417ac3f2080a1cefda5e9e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"e3bd4a02efe99f891912192a8188490059f0792d474be03435f8296210fbe5ae","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007c0de0\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-q4zkjf7prwjv\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a82356a5-5cf7-4422-8f08-898257e76006\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a82356a5-5cf7-4422-8f08-898257e76006\",\n \"name\": \"a82356a5-5cf7-4422-8f08-898257e76006\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:17.380507861Z\",\n \"endTime\": \"2026-09-22T14:01:24.867942166Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007c0de0\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-q4zkjf7prwjv\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a82356a5-5cf7-4422-8f08-898257e76006\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a82356a5-5cf7-4422-8f08-898257e76006\",\n \"name\": \"a82356a5-5cf7-4422-8f08-898257e76006\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:17.380507861Z\",\n \"endTime\": \"2026-09-22T14:01:24.867942166Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b8cf0e3538e2705f09d96f2aca376b51e05181ec4d857be5a6c9082b55854b0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"a24c34c210d135aa0d6e6ff7c3d0f3267f00c546b3e06d7da5efcbd1b4502e66","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-rld9nl\" with version 4.20.38 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ca4560\u003e: \n failed to create HCP cluster cluster-zstream-4-20-rld9nl: failed waiting for cluster=\"cluster-zstream-4-20-rld9nl\" in resourcegroup=\"rg-zstream-upgrade-4-20-5pqqs67q87zt\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa4828c5-ed08-491a-9850-88300b44f617\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa4828c5-ed08-491a-9850-88300b44f617\",\n \"name\": \"aa4828c5-ed08-491a-9850-88300b44f617\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:17.855427061Z\",\n \"endTime\": \"2026-09-22T14:01:24.824477819Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-rld9nl\" with version 4.20.38 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ca4560\u003e: \n failed to create HCP cluster cluster-zstream-4-20-rld9nl: failed waiting for cluster=\"cluster-zstream-4-20-rld9nl\" in resourcegroup=\"rg-zstream-upgrade-4-20-5pqqs67q87zt\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa4828c5-ed08-491a-9850-88300b44f617\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa4828c5-ed08-491a-9850-88300b44f617\",\n \"name\": \"aa4828c5-ed08-491a-9850-88300b44f617\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:17.855427061Z\",\n \"endTime\": \"2026-09-22T14:01:24.824477819Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bac467d84e91a0bf8c39060d95ca53de5a4e9c1b45fd7a7b3455ac0ed9347c49","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"44ebe74846f27c338b0020b629b93cac4c94febab64c3e4ea5e2dc4c13ac6911","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b6ba80\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-7nfnl9sxpnw4\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/db771643-cb7b-4400-b0be-f0453c7d0b78\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/db771643-cb7b-4400-b0be-f0453c7d0b78\",\n \"name\": \"db771643-cb7b-4400-b0be-f0453c7d0b78\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:06.730860572Z\",\n \"endTime\": \"2026-09-22T14:01:14.792387469Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b6ba80\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-7nfnl9sxpnw4\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/db771643-cb7b-4400-b0be-f0453c7d0b78\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/db771643-cb7b-4400-b0be-f0453c7d0b78\",\n \"name\": \"db771643-cb7b-4400-b0be-f0453c7d0b78\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:06.730860572Z\",\n \"endTime\": \"2026-09-22T14:01:14.792387469Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c61013a45711ed0213677018600d3274863de8bfa6f32b7ec441cb9bb6ceefa6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"2bdf2d474187f006f2f6858ad11138f1aca63f2cf4d0bc3da3af6070e6381cf5","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001242760\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-cqg4vcwtml8g\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b15f1682-d00f-480f-b189-4cfab6cb1f2d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b15f1682-d00f-480f-b189-4cfab6cb1f2d\",\n \"name\": \"b15f1682-d00f-480f-b189-4cfab6cb1f2d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:13.858451659Z\",\n \"endTime\": \"2026-09-22T14:01:14.879913195Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001242760\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-cqg4vcwtml8g\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b15f1682-d00f-480f-b189-4cfab6cb1f2d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b15f1682-d00f-480f-b189-4cfab6cb1f2d\",\n \"name\": \"b15f1682-d00f-480f-b189-4cfab6cb1f2d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:13.858451659Z\",\n \"endTime\": \"2026-09-22T14:01:14.879913195Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cf5c29c94fbcb35f8805dd55287405df332cc5ff36fa1dad9784f65123494d12","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"a0937d42fb845e19eba0547c56962838f60fa2e7d16ff0ed1e5bc2f5dc2708a7","test_name":"Fleet should have HCP resource requirements data","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:115]: Timed out after 1800.001s.\nHCP resource requirements were not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:105 with:\nDataCurrent condition must be True\nExpected\n \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:115]: Timed out after 1800.001s.\nHCP resource requirements were not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:105 with:\nDataCurrent condition must be True\nExpected\n \u003cbool\u003e: ...\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"96d801deab41ec98748c7082577ee746091d92728a5e786bd4eef6a4d1b2ca0f","failure_pattern":"DataCurrent condition must be True","search_query":"DataCurrent condition must be True"}},{"row_id":"d08a507ab4774d6f2074619e94aed260a3f7b5c1b0e03aef2e6bffdfdb7172ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"1a5d886d4d80c8ab33aeab161d42302f0898e33e7b32b74d3ae3ffecb4fdaf49","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000924cc0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-smvxtvkx87p2\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5181718b-c651-4b64-b71c-5dfd1c86ee96\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5181718b-c651-4b64-b71c-5dfd1c86ee96\",\n \"name\": \"5181718b-c651-4b64-b71c-5dfd1c86ee96\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:53.578618144Z\",\n \"endTime\": \"2026-09-22T14:01:54.769879975Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000924cc0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-smvxtvkx87p2\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5181718b-c651-4b64-b71c-5dfd1c86ee96\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5181718b-c651-4b64-b71c-5dfd1c86ee96\",\n \"name\": \"5181718b-c651-4b64-b71c-5dfd1c86ee96\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:53.578618144Z\",\n \"endTime\": \"2026-09-22T14:01:54.769879975Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d1b20e26458c0618f1b251953af7d514d005c1e2fbf0e43d0e98c15f0ce5b0f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"702eb05f62f599c9db1b0b8e344c4e364171cafc8fecb8a2a28b47a1b9dcb72e","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-qwtrff\" with version 4.22.13 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011f6780\u003e: \n failed to create HCP cluster cluster-zstream-4-22-qwtrff: failed waiting for cluster=\"cluster-zstream-4-22-qwtrff\" in resourcegroup=\"rg-zstream-upgrade-4-22-9ntddsd5mwjb\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f691bca0-b85d-45cf-8de4-554ba3049b83\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f691bca0-b85d-45cf-8de4-554ba3049b83\",\n \"name\": \"f691bca0-b85d-45cf-8de4-554ba3049b83\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:53.959725008Z\",\n \"endTime\": \"2026-09-22T14:01:54.970928051Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-qwtrff\" with version 4.22.13 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011f6780\u003e: \n failed to create HCP cluster cluster-zstream-4-22-qwtrff: failed waiting for cluster=\"cluster-zstream-4-22-qwtrff\" in resourcegroup=\"rg-zstream-upgrade-4-22-9ntddsd5mwjb\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f691bca0-b85d-45cf-8de4-554ba3049b83\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f691bca0-b85d-45cf-8de4-554ba3049b83\",\n \"name\": \"f691bca0-b85d-45cf-8de4-554ba3049b83\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:53.959725008Z\",\n \"endTime\": \"2026-09-22T14:01:54.970928051Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d6db55a410196d67d519d44e421b49bdc1b19ee36fc7a2501fa9771131cd1c52","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"87092b0de71dc07b55bc06c22f40f3a2eb74e067b0845903e93faecb3fea9761","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016e8c40\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-rjq4p7796qpn\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67f6727d-867f-44c6-99f9-65659d41500e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67f6727d-867f-44c6-99f9-65659d41500e\",\n \"name\": \"67f6727d-867f-44c6-99f9-65659d41500e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:11.313848844Z\",\n \"endTime\": \"2026-09-22T14:01:14.727166871Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016e8c40\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-rjq4p7796qpn\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67f6727d-867f-44c6-99f9-65659d41500e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/67f6727d-867f-44c6-99f9-65659d41500e\",\n \"name\": \"67f6727d-867f-44c6-99f9-65659d41500e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:11.313848844Z\",\n \"endTime\": \"2026-09-22T14:01:14.727166871Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d73e6427a0400ddfb3d2de3dbdacb935098afe68ce53ff403f891a6ba9409e3f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"d33a734fa6fa20e8c750af55af3edc29a07a36d78fad26f156f3236f9dcd07a6","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-qkvr8k\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009f85e0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-qkvr8k: failed waiting for cluster=\"cp-ystream-upgrade-5-0-qkvr8k\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-4n8dcjk4zbh7\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cff55541-6190-4c33-afc8-f052dfe5eacf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cff55541-6190-4c33-afc8-f052dfe5eacf\",\n \"name\": \"cff55541-6190-4c33-afc8-f052dfe5eacf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:11.583499194Z\",\n \"endTime\": \"2026-09-22T14:01:14.703665149Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-qkvr8k\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009f85e0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-qkvr8k: failed waiting for cluster=\"cp-ystream-upgrade-5-0-qkvr8k\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-4n8dcjk4zbh7\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cff55541-6190-4c33-afc8-f052dfe5eacf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cff55541-6190-4c33-afc8-f052dfe5eacf\",\n \"name\": \"cff55541-6190-4c33-afc8-f052dfe5eacf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:11.583499194Z\",\n \"endTime\": \"2026-09-22T14:01:14.703665149Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ea797e4a3ca9334644c4941a4574d52745b4e50688ee2cb7046cff82e672fa6b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"1d43ccfbc61f660474c97dcd9a29831dbbb576bcd4d61281856d22526f3e3dc2","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a72f40\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-rvlj9pd4g49j\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c57dc13f-c997-4327-af90-f5f1ab3c0f39\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c57dc13f-c997-4327-af90-f5f1ab3c0f39\",\n \"name\": \"c57dc13f-c997-4327-af90-f5f1ab3c0f39\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:58.722624927Z\",\n \"endTime\": \"2026-09-22T14:02:04.725282332Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a72f40\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-rvlj9pd4g49j\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c57dc13f-c997-4327-af90-f5f1ab3c0f39\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c57dc13f-c997-4327-af90-f5f1ab3c0f39\",\n \"name\": \"c57dc13f-c997-4327-af90-f5f1ab3c0f39\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:58.722624927Z\",\n \"endTime\": \"2026-09-22T14:02:04.725282332Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"faf38bb2b4c66dafee85b230f63b433c3c28ac111600c35073c5023a7d71a34f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"b31be570968d5cc95f200746d9e769721c920835a846af0f9e792531b00e0ac5","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-29hg8c at version 4.22.14\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d92c00\u003e: \n failed to create HCP cluster np-skip-minor-29hg8c: failed waiting for cluster=\"np-skip-minor-29hg8c\" in resourcegroup=\"rg-np-skip-minor-29hg8c-4dgpbcj69fqv\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/53fab93e-9091-4e72-a485-8d1c0068d6fb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/53fab93e-9091-4e72-a485-8d1c0068d6fb\",\n \"name\": \"53fab93e-9091-4e72-a485-8d1c0068d6fb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:33:08.629407608Z\",\n \"endTime\": \"2026-09-22T14:02:14.714971948Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-29hg8c at version 4.22.14\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d92c00\u003e: \n failed to create HCP cluster np-skip-minor-29hg8c: failed waiting for cluster=\"np-skip-minor-29hg8c\" in resourcegroup=\"rg-np-skip-minor-29hg8c-4dgpbcj69fqv\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/53fab93e-9091-4e72-a485-8d1c0068d6fb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/53fab93e-9091-4e72-a485-8d1c0068d6fb\",\n \"name\": \"53fab93e-9091-4e72-a485-8d1c0068d6fb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:33:08.629407608Z\",\n \"endTime\": \"2026-09-22T14:02:14.714971948Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fbfd7e273c67547f9a36b5b00de69ae99f73480f338613b7da16d433c23336e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"85afb00f76eebeafd5afd1181f0b8eccdf5062283ecc7bfc786cd8040de4cf44","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-krw7bcv4h9tp/cluster-candidate-5-0-npd8xs should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00104eb20\u003e: \n failed to create HCP cluster cluster-candidate-5-0-npd8xs: failed waiting for cluster=\"cluster-candidate-5-0-npd8xs\" in resourcegroup=\"rg-candidate-5-0-krw7bcv4h9tp\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/677c6d18-0a88-45ea-8eef-dcf3bcaf9526\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/677c6d18-0a88-45ea-8eef-dcf3bcaf9526\",\n \"name\": \"677c6d18-0a88-45ea-8eef-dcf3bcaf9526\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:01.537906113Z\",\n \"endTime\": \"2026-09-22T14:01:04.991194141Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-krw7bcv4h9tp/cluster-candidate-5-0-npd8xs should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00104eb20\u003e: \n failed to create HCP cluster cluster-candidate-5-0-npd8xs: failed waiting for cluster=\"cluster-candidate-5-0-npd8xs\" in resourcegroup=\"rg-candidate-5-0-krw7bcv4h9tp\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/677c6d18-0a88-45ea-8eef-dcf3bcaf9526\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/677c6d18-0a88-45ea-8eef-dcf3bcaf9526\",\n \"name\": \"677c6d18-0a88-45ea-8eef-dcf3bcaf9526\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:01.537906113Z\",\n \"endTime\": \"2026-09-22T14:01:04.991194141Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fe977d62767edddee3b047f45c904d64da8d13e5f1adb9624a42bbffffe0544d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"faf4c3955df8db29a640a24fb0d0dc9027282375fadc9d070d72fce2067bc0b9","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005747c0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-f5kx8sntpz47\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bfe16b2b-1ba0-4224-a47a-10f0686acafc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bfe16b2b-1ba0-4224-a47a-10f0686acafc\",\n \"name\": \"bfe16b2b-1ba0-4224-a47a-10f0686acafc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:33:57.252045262Z\",\n \"endTime\": \"2026-09-22T14:03:04.849972903Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005747c0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-f5kx8sntpz47\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bfe16b2b-1ba0-4224-a47a-10f0686acafc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bfe16b2b-1ba0-4224-a47a-10f0686acafc\",\n \"name\": \"bfe16b2b-1ba0-4224-a47a-10f0686acafc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:33:57.252045262Z\",\n \"endTime\": \"2026-09-22T14:03:04.849972903Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fecbfee74c63c4405c670799b1b5a56877ac00ebf217e3ceef93ebb6e4e724dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","failed_at":"e2e","signature_id":"96bea114bff0b91e68ec24f631e1236819bf685d8d5765c52b8154dc1a26568b","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000823be0\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-7p9grtkmqq95\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1340bd4e-1432-48c6-b9c6-1e08cde4b773\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1340bd4e-1432-48c6-b9c6-1e08cde4b773\",\n \"name\": \"1340bd4e-1432-48c6-b9c6-1e08cde4b773\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:04.276124571Z\",\n \"endTime\": \"2026-09-22T14:01:04.883826856Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000823be0\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-7p9grtkmqq95\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1340bd4e-1432-48c6-b9c6-1e08cde4b773\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1340bd4e-1432-48c6-b9c6-1e08cde4b773\",\n \"name\": \"1340bd4e-1432-48c6-b9c6-1e08cde4b773\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:32:04.276124571Z\",\n \"endTime\": \"2026-09-22T14:01:04.883826856Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":63,"failure_pattern_ids":["0cd69af8904472f73d329003c8c5db9d0e892958b3af0d73d1a1dee951ccff6d","6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","96d801deab41ec98748c7082577ee746091d92728a5e786bd4eef6a4d1b2ca0f","9a8854d173302655314e0bdcbc8411f9e276ffe0aadf4e25f0e0bad8f8dcb51d","d11e817743c9b7b223c4f7499ea10417bad7afb08d112eac38001716535808b0","f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":63,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102371297248940032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"c530dc60b8d6e25606f9f32861e6bf0caa14baec","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T12:15:42Z","started_at":"2026-09-22T12:15:42Z","completed_at":"2026-09-22T14:29:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T14:37:53.066371346Z","timing_metadata_checked_at":"2026-09-22T14:37:53.066371346Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T14:37:53.07138143Z","region_metadata_checked_at":"2026-09-22T14:37:53.07138143Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"12a16595c04f92d54c00999e6782f32cc7c03d8187bdf32181ab06566bbcc0a2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102371297248940032","occurred_at":"2026-09-22T12:15:42Z","failed_at":"e2e","signature_id":"51057cb8671ceec45046c3798f134700b77132ac6c113e3e7197b692a3a31053","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:88]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fb01e0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-22ntvrtddssz\" to finish creating: GET https://rp.j8940032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/242f24a5-8f64-4654-9b39-e30267e94ed1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/242f24a5-8f64-4654-9b39-e30267e94ed1\",\n \"name\": \"242f24a5-8f64-4654-9b39-e30267e94ed1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:01:52.830008497Z\",\n \"endTime\": \"2026-09-22T13:30:57.820386496Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 21m40s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:88]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fb01e0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-22ntvrtddssz\" to finish creating: GET https://rp.j8940032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/242f24a5-8f64-4654-9b39-e30267e94ed1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/242f24a5-8f64-4654-9b39-e30267e94ed1\",\n \"name\": \"242f24a5-8f64-4654-9b39-e30267e94ed1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:01:52.830008497Z\",\n \"endTime\": \"2026-09-22T13:30:57.820386496Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 21m40s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"16c3adef0193e7ada4c88025d8873902a1e9ac8fb0ae444cd17155cb15f9c6c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102371297248940032","occurred_at":"2026-09-22T12:15:42Z","failed_at":"alert","signature_id":"2f5c275c19af39d433c7a0ffe224f7e5c2c3268c6bbe4ea0fae334b7478f8fe6","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s) (1 unknown, 3 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T13:15:15Z\n Ended: 2026-09-22T13:39:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8940032-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t02sh3cjhqhk5sqr2356elqhegnu68u-v20261001\", pod=\"router-6c76dd549c-ds2wz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t02sh3cjhqhk5sqr2356elqhegnu68u-v20261001/router-6c76dd549c-ds2wz has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"3cd6632ad76b0450a02ae4ed12df38bf2d9dbe6e31075107e540936b09efd19f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102371297248940032","occurred_at":"2026-09-22T12:15:42Z","failed_at":"alert","signature_id":"7ec1bacfd5828bea1dddf53be5010f15e6e95874c8514734962d8e8dfced23fe","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T13:50:49Z\n Ended: 2026-09-22T14:01:52Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j8940032-svc\", component=\"frontend\", method=\"delete\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: More than 5% of frontend requests for delete /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j8940032-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}},{"row_id":"9ed6b3dc3da8efe9f76908788232f71951c01d7b367a193682f081084a681b14","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102371297248940032","occurred_at":"2026-09-22T12:15:42Z","failed_at":"e2e","signature_id":"b368b19d396235037e1a94307d4bc5fed0b37f15b10f9fdb4873cee59285f0a0","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc001b1a0a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-wwxhmhlsh7vt\" to finish deleting: GET https://rp.j8940032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bacc4e5a-7f20-4127-8aed-f8e207759e95\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bacc4e5a-7f20-4127-8aed-f8e207759e95\",\n \"name\": \"bacc4e5a-7f20-4127-8aed-f8e207759e95\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:46:58.861939308Z\",\n \"endTime\": \"2026-09-22T14:11:01.43159062Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t02sdao0ej27cdpe8l00d8ep8hru718 still exists (deletion dispatched at 2026-09-22T13:47:00Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc001b1a0a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-wwxhmhlsh7vt\" to finish deleting: GET https://rp.j8940032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bacc4e5a-7f20-4127-8aed-f8e207759e95\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bacc4e5a-7f20-4127-8aed-f8e207759e95\",\n \"name\": \"bacc4e5a-7f20-4127-8aed-f8e207759e95\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:46:58.861939308Z\",\n \"endTime\": \"2026-09-22T14:11:01.43159062Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t02sdao0ej27cdpe8l00d8ep8hru718 still exists (deletion dispatched at 2026-09-22T13:47:00Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102371239929581568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7009,"pr_state":"closed","pr_sha":"618b09676377afab5b54de6bc2aa79b79d0456fa","final_merged_sha":"c83bdabaf08bdb32938b4526f3e12a94c26f811c","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-22T12:15:38Z","started_at":"2026-09-22T12:15:38Z","completed_at":"2026-09-22T14:15:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T14:27:17.521056002Z","timing_metadata_checked_at":"2026-09-22T14:27:17.521056002Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T14:27:17.52421569Z","region_metadata_checked_at":"2026-09-22T14:27:17.52421569Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102367431941951488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T12:00:30Z","started_at":"2026-09-22T12:00:30Z","completed_at":"2026-09-22T14:13:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T14:27:16.640592633Z","timing_metadata_checked_at":"2026-09-22T14:27:16.640592633Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T14:27:16.646206712Z","region_metadata_checked_at":"2026-09-22T14:27:16.646206712Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"62c8ccb78a46f9c3c91580471eaefe1749ccedaef4aae659b16497bb123f5fb5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102367431941951488","occurred_at":"2026-09-22T12:00:30Z","failed_at":"alert","signature_id":"a76dfeabb77ec9c40d970f39e98e5c3d30889b460b25840fc635fa5a5c93c4b9","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T13:27:35Z\n Ended: 2026-09-22T13:40:37Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j1951488-svc\", component=\"frontend\", method=\"get\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} exceeded 1 second over the past 30 minutes (cluster ci01-j1951488-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}},{"row_id":"b7cc364f23aa685c8a9cc7b5b095d7fa706e8334666175b6db98b5c83efe0141","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102367431941951488","occurred_at":"2026-09-22T12:00:30Z","failed_at":"alert","signature_id":"a775ad6559303e11abdc6f06a0ac66c193334df7354f421e2e170c6c4c7e9368","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T13:57:41Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j1951488-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-hzln7\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-hzln7 (namespace arobit) on cluster ci01-j1951488-svc has averaged 126.1% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"e033029809e278075059cdb1393706896c74336f2f41e30e397f1416dbbb1072","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102367431941951488","occurred_at":"2026-09-22T12:00:30Z","failed_at":"e2e","signature_id":"dec9dde40f2f10efdc3071a06f16f401ec1f491f0a234e712f32edcadd53b386","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000c88840\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-784f6vzvhd54\" to finish deleting: GET https://rp.j1951488.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/22b142d3-4083-40a9-b2f7-0c8d747fb92b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/22b142d3-4083-40a9-b2f7-0c8d747fb92b\",\n \"name\": \"22b142d3-4083-40a9-b2f7-0c8d747fb92b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:22:36.19366526Z\",\n \"endTime\": \"2026-09-22T13:46:43.924656305Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t02o05k4mjacdppm5v91uu7di2s60fh still exists (deletion dispatched at 2026-09-22T13:22:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000c88840\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-784f6vzvhd54\" to finish deleting: GET https://rp.j1951488.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/22b142d3-4083-40a9-b2f7-0c8d747fb92b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/22b142d3-4083-40a9-b2f7-0c8d747fb92b\",\n \"name\": \"22b142d3-4083-40a9-b2f7-0c8d747fb92b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:22:36.19366526Z\",\n \"endTime\": \"2026-09-22T13:46:43.924656305Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t02o05k4mjacdppm5v91uu7di2s60fh still exists (deletion dispatched at 2026-09-22T13:22:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102367426107674624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6128,"pr_state":"open","pr_sha":"170f76835cc1a848764bf285f1e459b293c9fc67","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T12:00:29Z","started_at":"2026-09-22T12:00:29Z","completed_at":"2026-09-22T14:21:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T14:37:53.042522825Z","timing_metadata_checked_at":"2026-09-22T14:37:53.042522825Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T14:37:53.045871814Z","region_metadata_checked_at":"2026-09-22T14:37:53.045871814Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"047fc8ee7d60a21502a5b0dd158287825dbb17860f10efbcbd464588a3c4fbb1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102367426107674624","occurred_at":"2026-09-22T12:00:29Z","failed_at":"alert","signature_id":"6c8c17148df8d724ea7e8dac7eeed751ab50ce8837feab77bd06d0501cc3031f","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T13:20:51Z\n Ended: 2026-09-22T13:30:54Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7674624-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7674624-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"0bedbee2a70d5482490d025620c64ceca264a7ebd605fa22ec0f17d22ec58ff7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102367426107674624","occurred_at":"2026-09-22T12:00:29Z","failed_at":"e2e","signature_id":"34937e81704d7c1a456562c7e0f20bef69ca74160d4aa8539e4e3d2222a51ffc","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000bc0210\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-lpmmxlnr5f8b\" to finish deleting: GET https://rp.j7674624.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4d28fedd-a09c-4916-9a5b-65e9ecdf6442\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4d28fedd-a09c-4916-9a5b-65e9ecdf6442\",\n \"name\": \"4d28fedd-a09c-4916-9a5b-65e9ecdf6442\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:39:34.975442985Z\",\n \"endTime\": \"2026-09-22T14:03:35.724040601Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t02n2vj0cpu1967qjmj20dbkb8pajv6 still exists (deletion dispatched at 2026-09-22T13:39:35Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000bc0210\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-lpmmxlnr5f8b\" to finish deleting: GET https://rp.j7674624.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4d28fedd-a09c-4916-9a5b-65e9ecdf6442\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4d28fedd-a09c-4916-9a5b-65e9ecdf6442\",\n \"name\": \"4d28fedd-a09c-4916-9a5b-65e9ecdf6442\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:39:34.975442985Z\",\n \"endTime\": \"2026-09-22T14:03:35.724040601Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t02n2vj0cpu1967qjmj20dbkb8pajv6 still exists (deletion dispatched at 2026-09-22T13:39:35Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0dc1aafb16ab8290c10e568d485b265cf07d6073bbe2525362e1854a98ee0b58","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102367426107674624","occurred_at":"2026-09-22T12:00:29Z","failed_at":"e2e","signature_id":"928a4c5442ab7d0c4e8b9b1125e98edcd3463091d981041cf5ac778d3757533d","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a86060\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-dqw4tg\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-rpz8pwqkf6x2\" to finish deleting: GET https://rp.j7674624.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/59fe04db-687e-4883-94b3-d71e6b0e6949\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/59fe04db-687e-4883-94b3-d71e6b0e6949\",\n \"name\": \"59fe04db-687e-4883-94b3-d71e6b0e6949\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:23:25.69462366Z\",\n \"endTime\": \"2026-09-22T13:47:29.022299266Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t02qcvn03mmp1b6p4cv8m8otqvf9i4c still exists (deletion dispatched at 2026-09-22T13:23:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a86060\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-dqw4tg\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-rpz8pwqkf6x2\" to finish deleting: GET https://rp.j7674624.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/59fe04db-687e-4883-94b3-d71e6b0e6949\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/59fe04db-687e-4883-94b3-d71e6b0e6949\",\n \"name\": \"59fe04db-687e-4883-94b3-d71e6b0e6949\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:23:25.69462366Z\",\n \"endTime\": \"2026-09-22T13:47:29.022299266Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t02qcvn03mmp1b6p4cv8m8otqvf9i4c still exists (deletion dispatched at 2026-09-22T13:23:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2c86f6aa3a1af9bb98c2289e389590c59d67ee755fdd935b35cd00f40d8eb830","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102367426107674624","occurred_at":"2026-09-22T12:00:29Z","failed_at":"alert","signature_id":"ff9223586b6f06f638cd8392395aefcf5740b8c5345bdec7a43fdef9d217a7f4","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T13:59:56Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j7674624-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-8gnzs\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-8gnzs (namespace arobit) on cluster ci01-j7674624-svc has averaged 122.9% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102367395485061120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6995,"pr_state":"closed","pr_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","final_merged_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T12:00:21Z","started_at":"2026-09-22T12:00:21Z","completed_at":"2026-09-22T14:14:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T14:27:17.059848447Z","timing_metadata_checked_at":"2026-09-22T14:27:17.059848447Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T14:27:17.062848536Z","region_metadata_checked_at":"2026-09-22T14:27:17.062848536Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"26897e44fea6d7fd8d4b2eb19308ed8338dc8da58a364f510c5e1a9720e2380f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102367395485061120","occurred_at":"2026-09-22T12:00:21Z","failed_at":"e2e","signature_id":"97f5e8b0ca0a4de77ab6536bf9a3b7de05d88fdb06f441d1fed55e967b55be5b","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00186a078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-d6952r\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-2drztfzcz8ln\" to finish deleting: GET https://rp.j5061120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2b6d8126-6b4c-4de8-86ce-6d951164492f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2b6d8126-6b4c-4de8-86ce-6d951164492f\",\n \"name\": \"2b6d8126-6b4c-4de8-86ce-6d951164492f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:23:15.903664501Z\",\n \"endTime\": \"2026-09-22T13:47:15.971767798Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t02nnilq7ncv48hc3nk14tuqve9oq7s still exists (deletion dispatched at 2026-09-22T13:23:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00186a078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-d6952r\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-2drztfzcz8ln\" to finish deleting: GET https://rp.j5061120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2b6d8126-6b4c-4de8-86ce-6d951164492f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2b6d8126-6b4c-4de8-86ce-6d951164492f\",\n \"name\": \"2b6d8126-6b4c-4de8-86ce-6d951164492f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:23:15.903664501Z\",\n \"endTime\": \"2026-09-22T13:47:15.971767798Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t02nnilq7ncv48hc3nk14tuqve9oq7s still exists (deletion dispatched at 2026-09-22T13:23:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102367380872105984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6756,"pr_state":"open","pr_sha":"a65a65d0f5af3365546d2f648ce8be93984b79b0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T12:00:18Z","started_at":"2026-09-22T12:00:18Z","completed_at":"2026-09-22T14:06:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T14:16:21.761950644Z","timing_metadata_checked_at":"2026-09-22T14:16:21.761950644Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T14:16:21.765615831Z","region_metadata_checked_at":"2026-09-22T14:16:21.765615831Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"7247fab078d8db32b0468923a0d05087d42e2ef6706ce1487089230709c431dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102367380872105984","occurred_at":"2026-09-22T12:00:18Z","failed_at":"e2e","signature_id":"9fc712ec1800f8f44b8fa79c7cd63de7cea995fbb405e3ab6f51fbabdcb7b683","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:177]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b45e20\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-nvlgx5jfn6c7\" to finish creating: GET https://rp.j2105984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/59212393-2ed7-490f-a765-56b81be64c8e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/59212393-2ed7-490f-a765-56b81be64c8e\",\n \"name\": \"59212393-2ed7-490f-a765-56b81be64c8e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T12:57:41.103155101Z\",\n \"endTime\": \"2026-09-22T13:16:45.486929869Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:177]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b45e20\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-nvlgx5jfn6c7\" to finish creating: GET https://rp.j2105984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/59212393-2ed7-490f-a765-56b81be64c8e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/59212393-2ed7-490f-a765-56b81be64c8e\",\n \"name\": \"59212393-2ed7-490f-a765-56b81be64c8e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T12:57:41.103155101Z\",\n \"endTime\": \"2026-09-22T13:16:45.486929869Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b99b0285284d36b274200be0e9fdcdcdd480d308bc8110ae5abd507232aa011e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102367380872105984","occurred_at":"2026-09-22T12:00:18Z","failed_at":"alert","signature_id":"625c86184442105b29032ee621018bbc8b2cc587e8ad3237c8894fae23fd25be","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T12:56:44Z\n Ended: 2026-09-22T13:29:42Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j2105984-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j2105984-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"f32db3cc09267fbd36fd45711798f6410907bce4bd49394c663a72926225bdff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102367380872105984","occurred_at":"2026-09-22T12:00:18Z","failed_at":"e2e","signature_id":"9cd2d41b80cce088eec466c6b8d8c74e6008102a92ae17f83120148590a9efe2","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000a8a4e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-4nfw84mzx8tm\" to finish deleting: GET https://rp.j2105984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b61d39c7-956d-42aa-8f64-5073f11a84af\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b61d39c7-956d-42aa-8f64-5073f11a84af\",\n \"name\": \"b61d39c7-956d-42aa-8f64-5073f11a84af\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:22:25.901102704Z\",\n \"endTime\": \"2026-09-22T13:46:32.743102747Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t02mfa1v5gfqso6k7vcuahuapgr4h0p still exists (deletion dispatched at 2026-09-22T13:22:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000a8a4e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-4nfw84mzx8tm\" to finish deleting: GET https://rp.j2105984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b61d39c7-956d-42aa-8f64-5073f11a84af\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b61d39c7-956d-42aa-8f64-5073f11a84af\",\n \"name\": \"b61d39c7-956d-42aa-8f64-5073f11a84af\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:22:25.901102704Z\",\n \"endTime\": \"2026-09-22T13:46:32.743102747Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t02mfa1v5gfqso6k7vcuahuapgr4h0p still exists (deletion dispatched at 2026-09-22T13:22:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102366747628670976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6993,"pr_state":"open","pr_sha":"21a147211815925c29b0b72f7f28854abe4bee63","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T11:57:46Z","started_at":"2026-09-22T11:57:46Z","completed_at":"2026-09-22T13:53:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T14:05:51.58024546Z","timing_metadata_checked_at":"2026-09-22T14:05:51.58024546Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T14:05:51.58643004Z","region_metadata_checked_at":"2026-09-22T14:05:51.58643004Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"49b615e93c2643054250cbe657b9562b05e74e89f4eb5beb01674b7dc59d3e77","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102366747628670976","occurred_at":"2026-09-22T11:57:46Z","failed_at":"e2e","signature_id":"c5182d9a8aba3cc03529aeb383c02d293502a368a20ea2a1928a3c2c21a08902","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e9f240\u003e: \n failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-gmkmxkr8rtmb\" to finish creating: GET https://rp.j8670976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/24481b8a-04e4-437a-8420-e00fa62a404d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/24481b8a-04e4-437a-8420-e00fa62a404d\",\n \"name\": \"24481b8a-04e4-437a-8420-e00fa62a404d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:09:58.667800577Z\",\n \"endTime\": \"2026-09-22T13:29:05.886575623Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e9f240\u003e: \n failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-gmkmxkr8rtmb\" to finish creating: GET https://rp.j8670976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/24481b8a-04e4-437a-8420-e00fa62a404d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/24481b8a-04e4-437a-8420-e00fa62a404d\",\n \"name\": \"24481b8a-04e4-437a-8420-e00fa62a404d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:09:58.667800577Z\",\n \"endTime\": \"2026-09-22T13:29:05.886575623Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102366391112830976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6590,"pr_state":"open","pr_sha":"17723dd1fd8fdc29b226962e5a96f754eaced5b4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T11:56:22Z","started_at":"2026-09-22T11:56:22Z","completed_at":"2026-09-22T12:51:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T13:03:02.330478048Z","timing_metadata_checked_at":"2026-09-22T13:03:02.330478048Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T13:03:02.334039836Z","region_metadata_checked_at":"2026-09-22T13:03:02.334039836Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d5532d37569f2b3f095ebcbb1d43760ef847122d5dd507fcaf00b4385bdbe77c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102366391112830976","occurred_at":"2026-09-22T11:56:22Z","failed_at":"provision","signature_id":"4e96560f2ff6d93bac0f0e5cfb31586c4f9dc831e4a29778f4268da2aebff893","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-22T12:18:17.689Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-09-22T12:45:29.994Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-22T12:18:17.695906444Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2830976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2830976-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-09-22T12:18:20.00208003Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2830976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2830976-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-22T12:18:20.14445826Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2830976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2830976-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T12:45:29.992055155Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2830976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2830976-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"CF0FD6F1065F90B5\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2830976-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2830976-mgmt-2/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j2830976-mgmt-2/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-22T12:43:18.8605679Z\\\",\\\"duration\\\":\\\"PT8.5695733S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 44 for family StandardDdsv6Family for region centralus. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-22T12:45:29.992133756Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2830976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2830976-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"DD6EAF8731EA851B\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2830976-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2830976-mgmt-2/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j2830976-mgmt-2/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-22T12:43:18.8605679Z\\\",\\\"duration\\\":\\\"PT1M23.5357859S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T12:45:29.992147237Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2830976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2830976-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"9E2ABCCAB49D4C81\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2830976-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2830976-mgmt-2/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j2830976-mgmt-2/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-22T12:43:18.8605679Z\\\",\\\"duration\\\":\\\"PT1M23.3548981S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T12:45:29.992157287Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2830976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2830976-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"42D7531EF6508071\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2830976-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2830976-mgmt-2/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j2830976-mgmt-2/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-22T12:43:18.8605679Z\\\",\\\"duration\\\":\\\"PT1M53.0336442S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T12:45:29.992166697Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2830976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2830976-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"1B516DD7916AE405\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2830976-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2830976-mgmt-2/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j2830976-mgmt-2/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-22T12:18:26.1875166Z\\\",\\\"duration\\\":\\\"PT26M47.0458862S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T12:45:29.992178797Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2830976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2830976-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"2569996FCD48888B\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2830976-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2830976-mgmt-2\\\",\\\"resourceName\\\":\\\"ci01-j2830976-mgmt-2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-22T12:18:26.1875166Z\\\",\\\"duration\\\":\\\"PT24M52.6018784S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T12:45:29.992201978Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2830976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2830976-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2830976-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584115273851533055\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2830976-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2830976-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2830976-mgmt-2/agentPools/userswft3\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 44 for family StandardDdsv6Family for region centralus. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n }\\\\n ]\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"27m12.296304024s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e5f4977400f08751c42a094e098f25015a196161edfd5a441bd871ef2846ba3e","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ErrCode_InsufficientVCPUQuota; detail message Insufficient vcpu quota requested \u003ccount\u003e, remaining \u003ccount\u003e for family StandardDdsv6Family for region centralus. If you want to increase the quota, please follow this instruction: \u003curl\u003e Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e5f4977400f08751c42a094e098f25015a196161edfd5a441bd871ef2846ba3e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102359588845129728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7070,"pr_state":"closed","pr_sha":"10d1e6e67ad84132cdd528497cb78998e181d5f2","final_merged_sha":"10d1e6e67ad84132cdd528497cb78998e181d5f2","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T11:29:20Z","started_at":"2026-09-22T11:29:20Z","completed_at":"2026-09-22T13:48:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T13:55:48.45644082Z","timing_metadata_checked_at":"2026-09-22T13:55:48.45644082Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T13:55:48.460661706Z","region_metadata_checked_at":"2026-09-22T13:55:48.460661706Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"677e919e95917fa4abf3eb78d36daf904fd921847b5a5452546d5e8fb2df0589","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102359588845129728","occurred_at":"2026-09-22T11:29:20Z","failed_at":"e2e","signature_id":"7c852bc88bc03a1f6a63c0506c2a307d43133205030b3cd74f906a160ee3d56b","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00104c078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-hj7wxhq2hq8n\" to finish deleting: GET https://rp.j5129728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d29e61b2-26f8-48f8-8b4e-5d1a334405b3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d29e61b2-26f8-48f8-8b4e-5d1a334405b3\",\n \"name\": \"d29e61b2-26f8-48f8-8b4e-5d1a334405b3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:01:37.902956089Z\",\n \"endTime\": \"2026-09-22T13:25:46.268749052Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t02717lssolbvgogntqqtclnqlr0fpl still exists (deletion dispatched at 2026-09-22T13:01:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00104c078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-hj7wxhq2hq8n\" to finish deleting: GET https://rp.j5129728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d29e61b2-26f8-48f8-8b4e-5d1a334405b3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d29e61b2-26f8-48f8-8b4e-5d1a334405b3\",\n \"name\": \"d29e61b2-26f8-48f8-8b4e-5d1a334405b3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T13:01:37.902956089Z\",\n \"endTime\": \"2026-09-22T13:25:46.268749052Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t02717lssolbvgogntqqtclnqlr0fpl still exists (deletion dispatched at 2026-09-22T13:01:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9eed825109fa744d14d9cc5a67712f78cd24c375489e75c4577873befe614b4a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102359588845129728","occurred_at":"2026-09-22T11:29:20Z","failed_at":"alert","signature_id":"3491f500d38d72a5dade4517718158749b870cd35afca4fd6f9298720307de7d","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T13:26:35Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j5129728-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-7wvxd\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-7wvxd (namespace arobit) on cluster ci01-j5129728-svc has averaged 126.1% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102358117298737152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6752,"pr_state":"closed","pr_sha":"d433c21668cfdc66f98262f9099c8086b4397248","final_merged_sha":"d433c21668cfdc66f98262f9099c8086b4397248","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T11:23:29Z","started_at":"2026-09-22T11:23:29Z","completed_at":"2026-09-22T13:37:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T13:45:02.565906926Z","timing_metadata_checked_at":"2026-09-22T13:45:02.565906926Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T13:45:02.568712916Z","region_metadata_checked_at":"2026-09-22T13:45:02.568712916Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"aabfefa63e9a51356b7aad940f6e83b53f46e18cde23a754e8a255227cad741f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102358117298737152","occurred_at":"2026-09-22T11:23:29Z","failed_at":"e2e","signature_id":"b0c4616901a6dc97e982f9527c5d28cbd309863e5269f862500342c9b8619681","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00190c378\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-sfzxvlgpv6ph\" to finish deleting: GET https://rp.j8737152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d32a2314-3299-439e-8c20-7c7037b11098\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d32a2314-3299-439e-8c20-7c7037b11098\",\n \"name\": \"d32a2314-3299-439e-8c20-7c7037b11098\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T12:34:28.832612498Z\",\n \"endTime\": \"2026-09-22T12:58:35.458664282Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t022ukf70n48f1pegs95hv1qcaeki9p still exists (deletion dispatched at 2026-09-22T12:34:36Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00190c378\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-sfzxvlgpv6ph\" to finish deleting: GET https://rp.j8737152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d32a2314-3299-439e-8c20-7c7037b11098\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d32a2314-3299-439e-8c20-7c7037b11098\",\n \"name\": \"d32a2314-3299-439e-8c20-7c7037b11098\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T12:34:28.832612498Z\",\n \"endTime\": \"2026-09-22T12:58:35.458664282Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t022ukf70n48f1pegs95hv1qcaeki9p still exists (deletion dispatched at 2026-09-22T12:34:36Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d5332f240422b852a090b25549305f77b9311157a0e8127acc9cb448bd21c725","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102358117298737152","occurred_at":"2026-09-22T11:23:29Z","failed_at":"alert","signature_id":"32d007958a1c603d42d30427965ccdcaec85034f2288cd66f62776e68fbf4cac","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T13:14:30Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j8737152-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-7549w\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-7549w (namespace arobit) on cluster ci01-j8737152-svc has averaged 126% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"f80cc14d990e0ad78fbb4587a270e9e1ecc06e0a2fee52f185aea871fbb98f5a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102358117298737152","occurred_at":"2026-09-22T11:23:29Z","failed_at":"e2e","signature_id":"5626fe8511ed4c208a2b8f6e8fcb335eaa8a9082722db26ba8fcf76ecfb81810","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000012108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-65gqgkqpjknr\" to finish deleting: GET https://rp.j8737152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b89e8d10-3c6f-47c2-b094-8d090e0e8717\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b89e8d10-3c6f-47c2-b094-8d090e0e8717\",\n \"name\": \"b89e8d10-3c6f-47c2-b094-8d090e0e8717\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T12:54:20.279157709Z\",\n \"endTime\": \"2026-09-22T13:18:21.263130823Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t022sgna103g4j62q4jks87krqaalg9 still exists (deletion dispatched at 2026-09-22T12:54:21Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000012108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-65gqgkqpjknr\" to finish deleting: GET https://rp.j8737152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b89e8d10-3c6f-47c2-b094-8d090e0e8717\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b89e8d10-3c6f-47c2-b094-8d090e0e8717\",\n \"name\": \"b89e8d10-3c6f-47c2-b094-8d090e0e8717\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T12:54:20.279157709Z\",\n \"endTime\": \"2026-09-22T13:18:21.263130823Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t022sgna103g4j62q4jks87krqaalg9 still exists (deletion dispatched at 2026-09-22T12:54:21Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6092/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102353447767511040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6092,"pr_state":"open","pr_sha":"989af9a378d3bc8d27dffe56a5ccf18e79f5807e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T11:04:56Z","started_at":"2026-09-22T11:04:56Z","completed_at":"2026-09-22T13:09:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T13:23:43.335080263Z","timing_metadata_checked_at":"2026-09-22T13:23:43.335080263Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T13:23:43.339016851Z","region_metadata_checked_at":"2026-09-22T13:23:43.339016851Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"2cc4519a026a4254afbb53c7e138c6ffeae3e1561624454d02953c80f4c04c11","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6092/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102353447767511040","occurred_at":"2026-09-22T11:04:56Z","failed_at":"e2e","signature_id":"11a0a2dbbb69ddfa8609fe29df3b8e40cfdda6cce08442acfb43cebd7b72a660","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012202d0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-nj95pp\" in resourcegroup=\"rg-zstream-upgrade-4-20-zxqfccsdzw5x\" to finish deleting: GET https://rp.j7511040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ffa6556f-e169-447b-ba03-2ec23b8ebfcf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ffa6556f-e169-447b-ba03-2ec23b8ebfcf\",\n \"name\": \"ffa6556f-e169-447b-ba03-2ec23b8ebfcf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T12:26:07.876024314Z\",\n \"endTime\": \"2026-09-22T12:50:12.128002699Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t01u4bmgb6jfsk48mi3c59g61dc3ngm still exists (deletion dispatched at 2026-09-22T12:26:09Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012202d0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-nj95pp\" in resourcegroup=\"rg-zstream-upgrade-4-20-zxqfccsdzw5x\" to finish deleting: GET https://rp.j7511040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ffa6556f-e169-447b-ba03-2ec23b8ebfcf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ffa6556f-e169-447b-ba03-2ec23b8ebfcf\",\n \"name\": \"ffa6556f-e169-447b-ba03-2ec23b8ebfcf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T12:26:07.876024314Z\",\n \"endTime\": \"2026-09-22T12:50:12.128002699Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t01u4bmgb6jfsk48mi3c59g61dc3ngm still exists (deletion dispatched at 2026-09-22T12:26:09Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"447f8de303d0d7672927a51605ce9d2c4cfcac3494ef722c05c1088d01468b08","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6092/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102353447767511040","occurred_at":"2026-09-22T11:04:56Z","failed_at":"e2e","signature_id":"3bb4ede6806075a37b8c28d7b2a38f450e3de8181d81d31103a281b5545686b9","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:329]: failed to get admin REST config for cluster np-noedge-7z925l\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00077d780\u003e: \n failed waiting for hcpCluster=\"np-noedge-7z925l\" in resourcegroup=\"rg-np-noedge-7z925l-xwhx4zwbzvsx\" to finish getting creds: GET https://rp.j7511040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ac55f4d-25b7-4531-81f3-de46161013d9\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/8ac55f4d-25b7-4531-81f3-de46161013d9' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ac55f4d-25b7-4531-81f3-de46161013d9\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:329]: failed to get admin REST config for cluster np-noedge-7z925l\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00077d780\u003e: \n failed waiting for hcpCluster=\"np-noedge-7z925l\" in resourcegroup=\"rg-np-noedge-7z925l-xwhx4zwbzvsx\" to finish getting creds: GET https://rp.j7511040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ac55f4d-25b7-4531-81f3-de46161013d9\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/8ac55f4d-25b7-4531-81f3-de46161013d9' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ac55f4d-25b7-4531-81f3-de46161013d9\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102352929464782848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6922,"pr_state":"open","pr_sha":"87e40a07142a1457ed3f83374cc5d085c87d8f29","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T11:02:52Z","started_at":"2026-09-22T11:02:52Z","completed_at":"2026-09-22T13:22:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T13:34:32.083831112Z","timing_metadata_checked_at":"2026-09-22T13:34:32.083831112Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T13:34:32.0870688Z","region_metadata_checked_at":"2026-09-22T13:34:32.0870688Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"2384c2ccc91b2b35152d044a7b0f4cdd1044f2f1af6e2b17db104866746679dd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102352929464782848","occurred_at":"2026-09-22T11:02:52Z","failed_at":"e2e","signature_id":"01316794d9c0fb344341457a3d1310352434baeccefcfd1e361d50968abff590","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00129c048\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-rwcrtszf7jfv\" to finish deleting: GET https://rp.j4782848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6c98f5ee-3d3e-4d06-adb1-93b02a59b699\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6c98f5ee-3d3e-4d06-adb1-93b02a59b699\",\n \"name\": \"6c98f5ee-3d3e-4d06-adb1-93b02a59b699\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T12:40:11.897115045Z\",\n \"endTime\": \"2026-09-22T13:04:14.128066943Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t01t2m7o942c4sn0eema6t828rb1f1u still exists (deletion dispatched at 2026-09-22T12:40:14Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00129c048\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-rwcrtszf7jfv\" to finish deleting: GET https://rp.j4782848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6c98f5ee-3d3e-4d06-adb1-93b02a59b699\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6c98f5ee-3d3e-4d06-adb1-93b02a59b699\",\n \"name\": \"6c98f5ee-3d3e-4d06-adb1-93b02a59b699\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T12:40:11.897115045Z\",\n \"endTime\": \"2026-09-22T13:04:14.128066943Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t01t2m7o942c4sn0eema6t828rb1f1u still exists (deletion dispatched at 2026-09-22T12:40:14Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2e7ea4a2da5a989ac97e951e3ffed9958d538f4f365b4089539443a65919e93f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102352929464782848","occurred_at":"2026-09-22T11:02:52Z","failed_at":"e2e","signature_id":"d0f379246b4a2c91308e6d2f9e47ae91e7bb3d786e3223b6c8141724a6fb1201","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000d48120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-s5gwd7\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-4crbvttx4ddn\" to finish deleting: GET https://rp.j4782848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/efee312f-62e1-4b0e-aca7-6c7e60974ce7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/efee312f-62e1-4b0e-aca7-6c7e60974ce7\",\n \"name\": \"efee312f-62e1-4b0e-aca7-6c7e60974ce7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T12:24:03.210379981Z\",\n \"endTime\": \"2026-09-22T12:48:03.095196437Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t01t0n3nhj5mkrp4pcjj6062lg3lv8o still exists (deletion dispatched at 2026-09-22T12:24:04Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000d48120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-s5gwd7\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-4crbvttx4ddn\" to finish deleting: GET https://rp.j4782848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/efee312f-62e1-4b0e-aca7-6c7e60974ce7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/efee312f-62e1-4b0e-aca7-6c7e60974ce7\",\n \"name\": \"efee312f-62e1-4b0e-aca7-6c7e60974ce7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T12:24:03.210379981Z\",\n \"endTime\": \"2026-09-22T12:48:03.095196437Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t01t0n3nhj5mkrp4pcjj6062lg3lv8o still exists (deletion dispatched at 2026-09-22T12:24:04Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9bbe191475008f4c593784fe3c3e1bd6e9423816ab0f228fd864b0ee87fa2749","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102352929464782848","occurred_at":"2026-09-22T11:02:52Z","failed_at":"e2e","signature_id":"c5b55b9526fcca16eb255c0896e3a68835960f08ba55ea57d84d15dc778d8456","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001b50378\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-87wgx5\" in resourcegroup=\"rg-np-version-upgrade-87wgx5-pcmbclp6gkrs\" to finish deleting: GET https://rp.j4782848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2d459377-0fed-4f30-8665-439104478418\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2d459377-0fed-4f30-8665-439104478418\",\n \"name\": \"2d459377-0fed-4f30-8665-439104478418\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T12:33:05.196504994Z\",\n \"endTime\": \"2026-09-22T12:57:14.101136035Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t01t0nogib9ejpsse48iaemplhqusib still exists (deletion dispatched at 2026-09-22T12:33:06Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001b50378\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-87wgx5\" in resourcegroup=\"rg-np-version-upgrade-87wgx5-pcmbclp6gkrs\" to finish deleting: GET https://rp.j4782848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2d459377-0fed-4f30-8665-439104478418\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2d459377-0fed-4f30-8665-439104478418\",\n \"name\": \"2d459377-0fed-4f30-8665-439104478418\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T12:33:05.196504994Z\",\n \"endTime\": \"2026-09-22T12:57:14.101136035Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t01t0nogib9ejpsse48iaemplhqusib still exists (deletion dispatched at 2026-09-22T12:33:06Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9ce7baba6661d6976da2504de8f828406be46c5d6276137180b5f9e9cee0f5b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102352929464782848","occurred_at":"2026-09-22T11:02:52Z","failed_at":"e2e","signature_id":"e9fae8337be48ee3b5d096071bada7dbec389c7f5d2bbfed662ea298bb36fe32","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001990210\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-bvfmtf6bf2br\" to finish deleting: GET https://rp.j4782848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6d2c27ff-6119-47a9-b767-c54c12209ed2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6d2c27ff-6119-47a9-b767-c54c12209ed2\",\n \"name\": \"6d2c27ff-6119-47a9-b767-c54c12209ed2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T12:20:47.737752758Z\",\n \"endTime\": \"2026-09-22T12:44:53.098039041Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t01sq695pk0nqm2fvfsrqu4l67jje6m still exists (deletion dispatched at 2026-09-22T12:20:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001990210\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-bvfmtf6bf2br\" to finish deleting: GET https://rp.j4782848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6d2c27ff-6119-47a9-b767-c54c12209ed2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6d2c27ff-6119-47a9-b767-c54c12209ed2\",\n \"name\": \"6d2c27ff-6119-47a9-b767-c54c12209ed2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T12:20:47.737752758Z\",\n \"endTime\": \"2026-09-22T12:44:53.098039041Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t01sq695pk0nqm2fvfsrqu4l67jje6m still exists (deletion dispatched at 2026-09-22T12:20:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102351739108724736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7009,"pr_state":"closed","pr_sha":"618b09676377afab5b54de6bc2aa79b79d0456fa","final_merged_sha":"c83bdabaf08bdb32938b4526f3e12a94c26f811c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T10:58:09Z","started_at":"2026-09-22T10:58:09Z","completed_at":"2026-09-22T11:59:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T12:11:16.809297331Z","timing_metadata_checked_at":"2026-09-22T12:11:16.809297331Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T12:11:16.812632618Z","region_metadata_checked_at":"2026-09-22T12:11:16.812632618Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ea39abd7c9b7f7c736aa7500a9ff4c4a0a2adfc0b058ddfcc5b9d4630a06e46c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102351739108724736","occurred_at":"2026-09-22T10:58:09Z","failed_at":"provision","signature_id":"2feb22461dc45624d789f70eb42c73cb09a2352e5d415e8876b9a64e3920abfd","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-22T11:23:53.670Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-09-22T11:53:53.299Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-22T11:23:53.695872086Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8724736-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j8724736-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\"}\\n{\\\"time\\\":\\\"2026-09-22T11:23:55.867241647Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8724736-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j8724736-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-22T11:23:55.995607642Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8724736-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j8724736-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"70097b1d8f359ab8e0f51f7aa2fbee8af0018247211e97b38af99aca36140186","failure_pattern":"error running Shell Step, failed to execute shell command: {\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"\u003credacted-id\u003e\",\"resourceGroup\":\"hcp-underlay-ci01-j8724736-mgmt-2\",\"clusterName\":\"ci01-j872473","search_query":"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-22T11:23:53.695872086Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8724736-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j8724736-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\"}\\n{\\\"time\\\":\\\"2026-09-22T11:23:55.867241647Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8724736-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j8724736-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-22T11:23:55.995607642Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8724736-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j8724736-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["70097b1d8f359ab8e0f51f7aa2fbee8af0018247211e97b38af99aca36140186"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7067/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102340054952185856","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7067,"pr_state":"closed","pr_sha":"8ae5d545296859679e6c957bce4ca9e5af6e376c","final_merged_sha":"8ae5d545296859679e6c957bce4ca9e5af6e376c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T10:11:43Z","started_at":"2026-09-22T10:11:43Z","completed_at":"2026-09-22T12:21:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T12:31:24.596130139Z","timing_metadata_checked_at":"2026-09-22T12:31:24.596130139Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T12:31:24.599899527Z","region_metadata_checked_at":"2026-09-22T12:31:24.599899527Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"1819653132620fae07c93443e1da2410a2bf6cd6e20054796ab5554efd8eea19","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7067/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102340054952185856","occurred_at":"2026-09-22T10:11:43Z","failed_at":"e2e","signature_id":"4afad7a628492d65a31b44f111c2f91a70c0ac311120930e8d36551c21514ccb","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00106c0a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-8jmmzcb9t2nx\" to finish deleting: GET https://rp.j2185856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f4c77cde-689d-4991-88ee-14b0517d1495\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f4c77cde-689d-4991-88ee-14b0517d1495\",\n \"name\": \"f4c77cde-689d-4991-88ee-14b0517d1495\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T11:38:59.29364075Z\",\n \"endTime\": \"2026-09-22T12:03:02.197932791Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t010oqmj8rpakhnf6q2d5om47fbernn still exists (deletion dispatched at 2026-09-22T11:39:00Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00106c0a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-8jmmzcb9t2nx\" to finish deleting: GET https://rp.j2185856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f4c77cde-689d-4991-88ee-14b0517d1495\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f4c77cde-689d-4991-88ee-14b0517d1495\",\n \"name\": \"f4c77cde-689d-4991-88ee-14b0517d1495\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T11:38:59.29364075Z\",\n \"endTime\": \"2026-09-22T12:03:02.197932791Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t010oqmj8rpakhnf6q2d5om47fbernn still exists (deletion dispatched at 2026-09-22T11:39:00Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102338247882444800","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7009,"pr_state":"closed","pr_sha":"618b09676377afab5b54de6bc2aa79b79d0456fa","final_merged_sha":"c83bdabaf08bdb32938b4526f3e12a94c26f811c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T10:04:32Z","started_at":"2026-09-22T10:04:32Z","completed_at":"2026-09-22T10:49:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T11:07:46.879401696Z","timing_metadata_checked_at":"2026-09-22T11:07:46.879401696Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T11:07:46.882201186Z","region_metadata_checked_at":"2026-09-22T11:07:46.882201186Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"4d7cab0844193ff4593aed1fefa0c9e75a9baea501336b77018dee24f8c654ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102338247882444800","occurred_at":"2026-09-22T10:04:32Z","failed_at":"provision","signature_id":"3096746f308f4ed95275075880d60cb754e454faacf298548b21a679d504f7a7","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-22T10:35:24.276Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-09-22T10:42:31.669Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-22T10:35:24.282791483Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2444800-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2444800-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\"}\\n{\\\"time\\\":\\\"2026-09-22T10:35:26.284841392Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2444800-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2444800-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-22T10:35:26.448708771Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2444800-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2444800-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T10:42:31.666852933Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2444800-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2444800-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"48630ADF8315F9F4\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2444800-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2444800-mgmt-2/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j2444800-mgmt-2/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-22T10:40:38.6323694Z\\\",\\\"duration\\\":\\\"PT1M40.6270261S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T10:42:31.666918654Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2444800-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2444800-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"50E367DC062B740B\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2444800-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2444800-mgmt-2/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j2444800-mgmt-2/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-22T10:40:38.6323694Z\\\",\\\"duration\\\":\\\"PT1M39.3479577S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T10:42:31.666940194Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2444800-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2444800-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"96EF32884A17F314\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2444800-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2444800-mgmt-2/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j2444800-mgmt-2/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-22T10:40:38.6323694Z\\\",\\\"duration\\\":\\\"PT4.1058994S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-22T10:42:31.666978395Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2444800-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2444800-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"1E29754134C2A673\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2444800-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2444800-mgmt-2/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j2444800-mgmt-2/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-22T10:35:30.2867656Z\\\",\\\"duration\\\":\\\"PT5M11.8733849S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-22T10:42:31.667019736Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2444800-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2444800-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"DD00B42016EFC1CA\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2444800-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2444800-mgmt-2/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j2444800-mgmt-2/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-22T10:40:38.6323694Z\\\",\\\"duration\\\":\\\"PT3.4027868S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-22T10:42:31.667063427Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2444800-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2444800-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"3962D5B17452A0C8\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2444800-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2444800-mgmt-2\\\",\\\"resourceName\\\":\\\"ci01-j2444800-mgmt-2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-22T10:35:30.2867656Z\\\",\\\"duration\\\":\\\"PT5M8.2796787S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T10:42:31.667099327Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2444800-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2444800-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2444800-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584115335588388698\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2444800-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2444800-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2444800-mgmt-2/agentPools/userswft2\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n },\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2444800-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2444800-mgmt-2/agentPools/userswft1\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n },\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2444800-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2444800-mgmt-2/agentPools/userswft3\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n }\\\\n ]\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"7m7.384317605s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cd0b0b8f7ac9ea6402ca4eac6d805c330783a014b8096e0958d5137c2c282bdb","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ErrCode_InsufficientVCPUQuota; detail message Insufficient vcpu quota requested \u003ccount\u003e, remaining \u003ccount\u003e for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: \u003curl\u003e Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["cd0b0b8f7ac9ea6402ca4eac6d805c330783a014b8096e0958d5137c2c282bdb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7066/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102337463992193024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7066,"pr_state":"closed","pr_sha":"eca364775bafae8fcd8399b7d0db91dc6fb25c6d","final_merged_sha":"23179a5354aa06a57efca1f82118702c3e475ce6","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T10:01:02Z","started_at":"2026-09-22T10:01:02Z","completed_at":"2026-09-22T12:28:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T12:41:54.49771142Z","timing_metadata_checked_at":"2026-09-22T12:41:54.49771142Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T12:41:54.50064961Z","region_metadata_checked_at":"2026-09-22T12:41:54.50064961Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"40269088910ade2a3cf17599a58d739d1c74aff1050927471024396bcced5494","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7066/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102337463992193024","occurred_at":"2026-09-22T10:01:02Z","failed_at":"alert","signature_id":"097bb7e262a51161f0f6deb237e175dc5d5fdb158c34757a9099308fac85df34","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T11:05:13Z\n Ended: 2026-09-22T11:38:14Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j2193024-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j2193024-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"67525aa9cc80b70af0051afb462e6b21bc4cb5798c41e1b7aa921bd591244e70","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7066/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102337463992193024","occurred_at":"2026-09-22T10:01:02Z","failed_at":"alert","signature_id":"cefa57c82ab99c9bb1b4362ad54a67403b423ec2ce56cd2a82667fd307603621","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T12:10:47Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j2193024-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-nffxd\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-nffxd (namespace arobit) on cluster ci01-j2193024-svc has averaged 126.2% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"83ba8e65efded89104cef7d933ec9249541d2783472abffeb5e583a1a4e36842","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7066/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102337463992193024","occurred_at":"2026-09-22T10:01:02Z","failed_at":"alert","signature_id":"4d9dcd6dafc17dff069e71dad10c9a950f19ca6ce800ac8f8f06506ff5cdcc58","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatency6h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T11:40:32Z\n Severity: Sev3\n Labels: alertname=\"userJourneyFrontendLatency6h30m\", cluster=\"ci01-j2193024-svc\", component=\"slo\", long_window=\"6h\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n Description: More than 6% of Frontend requests on cluster ci01-j2193024-svc exceeded 1s on both the 6h and 30m windows (6x burn of the 1% latency budget).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"addfa32c4c8681e954bead4d63f7ee51babad93b3eb924950aeb173e9712942b","failure_pattern":"alert [svc] userJourneyFrontendLatency6h30m fired","search_query":"alert [svc] userJourneyFrontendLatency6h30m fired"}},{"row_id":"85b15481ac99b47344b42ba654e680ccaf67c61db30b690e1624dc682d1673c7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7066/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102337463992193024","occurred_at":"2026-09-22T10:01:02Z","failed_at":"e2e","signature_id":"320ab412a4e2804f8ccd6d79e9bdb48fcd873437f28371c785640198716dfcc6","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001130b58\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-c9mwnv9mdv9l\" to finish deleting: GET https://rp.j2193024.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2b236a4e-5c28-455a-a3b8-408d1c8a5832\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2b236a4e-5c28-455a-a3b8-408d1c8a5832\",\n \"name\": \"2b236a4e-5c28-455a-a3b8-408d1c8a5832\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T11:43:46.417280075Z\",\n \"endTime\": \"2026-09-22T12:07:51.315583112Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t012ocf0lcq23fakt2qrpvp9htm8g26 still exists (deletion dispatched at 2026-09-22T11:43:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001130b58\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-c9mwnv9mdv9l\" to finish deleting: GET https://rp.j2193024.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2b236a4e-5c28-455a-a3b8-408d1c8a5832\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2b236a4e-5c28-455a-a3b8-408d1c8a5832\",\n \"name\": \"2b236a4e-5c28-455a-a3b8-408d1c8a5832\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T11:43:46.417280075Z\",\n \"endTime\": \"2026-09-22T12:07:51.315583112Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t012ocf0lcq23fakt2qrpvp9htm8g26 still exists (deletion dispatched at 2026-09-22T11:43:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b7dfeef62c2c038c15c1a2971a10b78c9b08076d72dafba93b71203459e6ff78","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7066/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102337463992193024","occurred_at":"2026-09-22T10:01:02Z","failed_at":"alert","signature_id":"b398e3b4b60ce0bfe92ea3c19658030c4ec5dee4a44f8a5f5c5664c590af5113","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T11:14:13Z\n Ended: 2026-09-22T11:26:09Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j2193024-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-09-22T11:24:10Z\n Ended: 2026-09-22T11:34:10Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j2193024-svc\", component=\"backend\", name=\"csstatedump\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"bb01637380e50c4b7766454ef54369d7ef552036b77feff75edfcef3bc15398d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7066/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102337463992193024","occurred_at":"2026-09-22T10:01:02Z","failed_at":"alert","signature_id":"2b6f0a14ed8cae2c4deeed2fde914673ffa2aeea1f351712769b94ee40fff620","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T11:41:13Z\n Ended: 2026-09-22T11:50:11Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j2193024-svc\", component=\"frontend\", method=\"get\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters exceeded 1 second over the past 30 minutes (cluster ci01-j2193024-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","addfa32c4c8681e954bead4d63f7ee51babad93b3eb924950aeb173e9712942b","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102337461471416320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6503,"pr_state":"closed","pr_sha":"6c5cfd09b9bbf1585b1cd37220da16677f327452","final_merged_sha":"a0bf660bfdf32222374a82406e0907cae4b53869","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T10:01:00Z","started_at":"2026-09-22T10:01:00Z","completed_at":"2026-09-22T12:26:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T12:41:54.335722996Z","timing_metadata_checked_at":"2026-09-22T12:41:54.335722996Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T12:41:54.339228184Z","region_metadata_checked_at":"2026-09-22T12:41:54.339228184Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"5bd4d9858349a1206c2753f034fd2469a70283b82f8b3e64c6c0d2dd1c27d6f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102337461471416320","occurred_at":"2026-09-22T10:01:00Z","failed_at":"e2e","signature_id":"8d08eca2a140bf94fe932d5a0d7fe9fea9199044029dcea39bb14577310f4851","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001234210\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-7lcrh4lksx7n\" to finish deleting: GET https://rp.j1416320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/11de3a89-d21d-46a0-8fe0-caf4a5581e21\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/11de3a89-d21d-46a0-8fe0-caf4a5581e21\",\n \"name\": \"11de3a89-d21d-46a0-8fe0-caf4a5581e21\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T11:44:22.052336204Z\",\n \"endTime\": \"2026-09-22T12:08:28.455213082Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t011ljgaaddoqvpihph8n2gttcgf40d still exists (deletion dispatched at 2026-09-22T11:44:23Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001234210\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-7lcrh4lksx7n\" to finish deleting: GET https://rp.j1416320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/11de3a89-d21d-46a0-8fe0-caf4a5581e21\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/11de3a89-d21d-46a0-8fe0-caf4a5581e21\",\n \"name\": \"11de3a89-d21d-46a0-8fe0-caf4a5581e21\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T11:44:22.052336204Z\",\n \"endTime\": \"2026-09-22T12:08:28.455213082Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t011ljgaaddoqvpihph8n2gttcgf40d still exists (deletion dispatched at 2026-09-22T11:44:23Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102336410580160512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7068,"pr_state":"open","pr_sha":"5d483559deea65ed903f146fc4f20cbbc522559d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-22T09:57:14Z","started_at":"2026-09-22T09:57:14Z","completed_at":"2026-09-22T12:08:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T12:21:18.745610186Z","timing_metadata_checked_at":"2026-09-22T12:21:18.745610186Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T12:21:18.75019837Z","region_metadata_checked_at":"2026-09-22T12:21:18.75019837Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102336356570107904","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6688,"pr_state":"open","pr_sha":"f8180cd6044fd44b58d22fb50e2a7d9429087dfe","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T09:57:01Z","started_at":"2026-09-22T09:57:01Z","completed_at":"2026-09-22T12:32:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T12:41:54.474690402Z","timing_metadata_checked_at":"2026-09-22T12:41:54.474690402Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T12:41:54.478450689Z","region_metadata_checked_at":"2026-09-22T12:41:54.478450689Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"6513db1fff9de33e2c309693ae1c56fec0f28bee695decff48e2f724c251244a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102336356570107904","occurred_at":"2026-09-22T09:57:01Z","failed_at":"alert","signature_id":"a73ccba8a620f8479e904c1b860b84849b279a7a78182d97426a6810dde93c5a","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T12:00:17Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j0107904-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-vw9hz\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-vw9hz (namespace arobit) on cluster ci01-j0107904-svc has averaged 127.7% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"d7ffe48429608ddc591637e6a41569415210f8e9adff29967b7cce3b24b9abdc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102336356570107904","occurred_at":"2026-09-22T09:57:01Z","failed_at":"e2e","signature_id":"db02d20b216e98887346005e3d003ac57a48a9edfe9a37448a2d26d11e0bffc6","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0021de108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-4ztvzt\" in resourcegroup=\"rg-np-version-upgrade-4ztvzt-vtjknvn9vdn5\" to finish deleting: GET https://rp.j0107904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2df4b80c-429b-4a34-8f6b-ddc0ca6cee78\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2df4b80c-429b-4a34-8f6b-ddc0ca6cee78\",\n \"name\": \"2df4b80c-429b-4a34-8f6b-ddc0ca6cee78\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T11:52:44.144339328Z\",\n \"endTime\": \"2026-09-22T12:16:46.994550731Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t016uc314bu49dg4lc99gfogfb8nvr9 still exists (deletion dispatched at 2026-09-22T11:52:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0021de108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-4ztvzt\" in resourcegroup=\"rg-np-version-upgrade-4ztvzt-vtjknvn9vdn5\" to finish deleting: GET https://rp.j0107904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2df4b80c-429b-4a34-8f6b-ddc0ca6cee78\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2df4b80c-429b-4a34-8f6b-ddc0ca6cee78\",\n \"name\": \"2df4b80c-429b-4a34-8f6b-ddc0ca6cee78\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T11:52:44.144339328Z\",\n \"endTime\": \"2026-09-22T12:16:46.994550731Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t016uc314bu49dg4lc99gfogfb8nvr9 still exists (deletion dispatched at 2026-09-22T11:52:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102333755334070272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6439,"pr_state":"closed","pr_sha":"8e51fd56182fb53110559c6686373da1a4435f34","final_merged_sha":"8e51fd56182fb53110559c6686373da1a4435f34","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T09:46:41Z","started_at":"2026-09-22T09:46:41Z","completed_at":"2026-09-22T10:46:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T10:56:57.593315157Z","timing_metadata_checked_at":"2026-09-22T10:56:57.593315157Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T10:56:57.596602146Z","region_metadata_checked_at":"2026-09-22T10:56:57.596602146Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"df907c4d431d217c6dec97d5ba72f7eecaee18c8c3a7d0039e9802b105c518de","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102333755334070272","occurred_at":"2026-09-22T09:46:41Z","failed_at":"provision","signature_id":"56583212f49aa2bec3088ae2a65589f7c4f6c467661d1f7190268818dac419c1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-22T10:31:08.961Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-09-22T10:38:47.425Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-22T10:31:08.979907747Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4070272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4070272-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\"}\\n{\\\"time\\\":\\\"2026-09-22T10:31:11.06396002Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4070272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4070272-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-22T10:31:11.22661971Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4070272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4070272-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T10:38:47.423846978Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4070272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4070272-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"F61EEEEBCFA56868\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4070272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4070272-mgmt-2/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j4070272-mgmt-2/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-22T10:36:41.5611028Z\\\",\\\"duration\\\":\\\"PT1M39.7596603S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T10:38:47.423898879Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4070272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4070272-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"8E07C9049ECB8DEF\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4070272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4070272-mgmt-2/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j4070272-mgmt-2/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-22T10:36:41.5611028Z\\\",\\\"duration\\\":\\\"PT1M39.7909151S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T10:38:47.423911859Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4070272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4070272-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"C8376C88B6FADF35\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4070272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4070272-mgmt-2/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j4070272-mgmt-2/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-22T10:36:41.5611028Z\\\",\\\"duration\\\":\\\"PT4.063355S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 18 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-22T10:38:47.42394617Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4070272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4070272-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"2DFDEDAE29CB0E2C\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4070272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4070272-mgmt-2/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j4070272-mgmt-2/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-22T10:36:41.5611028Z\\\",\\\"duration\\\":\\\"PT3.9227798S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 18 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-22T10:38:47.423968521Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4070272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4070272-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"04012E211BF318FE\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4070272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4070272-mgmt-2/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j4070272-mgmt-2/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-22T10:31:16.3138337Z\\\",\\\"duration\\\":\\\"PT5M28.8419351S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 18 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-22T10:38:47.424009742Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4070272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4070272-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"F96D9B5A6875A606\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4070272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4070272-mgmt-2\\\",\\\"resourceName\\\":\\\"ci01-j4070272-mgmt-2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-22T10:31:16.3138337Z\\\",\\\"duration\\\":\\\"PT5M25.0918247S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T10:38:47.424030272Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4070272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4070272-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4070272-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584115338140513340\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4070272-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4070272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4070272-mgmt-2/agentPools/userswft1\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 18 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n },\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4070272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4070272-mgmt-2/agentPools/userswft2\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 18 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n },\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4070272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4070272-mgmt-2/agentPools/userswft3\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 18 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n }\\\\n ]\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"7m38.444136305s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cd0b0b8f7ac9ea6402ca4eac6d805c330783a014b8096e0958d5137c2c282bdb","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ErrCode_InsufficientVCPUQuota; detail message Insufficient vcpu quota requested \u003ccount\u003e, remaining \u003ccount\u003e for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: \u003curl\u003e Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["cd0b0b8f7ac9ea6402ca4eac6d805c330783a014b8096e0958d5137c2c282bdb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7031/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102329484735680512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7031,"pr_state":"open","pr_sha":"96d978e0b776858d8164e58527dcd35c66a311e4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T09:29:43Z","started_at":"2026-09-22T09:29:43Z","completed_at":"2026-09-22T11:31:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T11:50:03.250271744Z","timing_metadata_checked_at":"2026-09-22T11:50:03.250271744Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T11:50:03.253063534Z","region_metadata_checked_at":"2026-09-22T11:50:03.253063534Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d60f4623a073df7e51a45c5059a2a312255eceee46ae63e61f4beea030ce6f84","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7031/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102329484735680512","occurred_at":"2026-09-22T09:29:43Z","failed_at":"e2e","signature_id":"53ed55054c50d93669e63983143c3a224f81ef6552fcf02580387d52aff966af","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00194c2e8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-downgrade-wqtjvn\" in resourcegroup=\"rg-np-downgrade-wqtjvn-6zvp2wkzd4qs\" to finish deleting: GET https://rp.j5680512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a4570083-37da-40f8-a2e2-fca3e9ade304\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a4570083-37da-40f8-a2e2-fca3e9ade304\",\n \"name\": \"a4570083-37da-40f8-a2e2-fca3e9ade304\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T10:49:11.895073397Z\",\n \"endTime\": \"2026-09-22T11:13:18.36225794Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t00dlbbm98iflefquha48s4p3gue5g9 still exists (deletion dispatched at 2026-09-22T10:49:12Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00194c2e8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-downgrade-wqtjvn\" in resourcegroup=\"rg-np-downgrade-wqtjvn-6zvp2wkzd4qs\" to finish deleting: GET https://rp.j5680512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a4570083-37da-40f8-a2e2-fca3e9ade304\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a4570083-37da-40f8-a2e2-fca3e9ade304\",\n \"name\": \"a4570083-37da-40f8-a2e2-fca3e9ade304\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T10:49:11.895073397Z\",\n \"endTime\": \"2026-09-22T11:13:18.36225794Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t00dlbbm98iflefquha48s4p3gue5g9 still exists (deletion dispatched at 2026-09-22T10:49:12Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102324260725854208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7041,"pr_state":"open","pr_sha":"383b3840a43a95f99f5e04e35ba2b1a3a9ff774b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T09:08:57Z","started_at":"2026-09-22T09:08:57Z","completed_at":"2026-09-22T11:17:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T11:28:40.406574821Z","timing_metadata_checked_at":"2026-09-22T11:28:40.406574821Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T11:28:40.411203606Z","region_metadata_checked_at":"2026-09-22T11:28:40.411203606Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"a066d2d7d352a537786bd03764bdb1ff499aeb0b01be5a9a54252d9744f54094","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102324260725854208","occurred_at":"2026-09-22T09:08:57Z","failed_at":"alert","signature_id":"214043246b33b8bd15f594a10236827f03be2665055b40a9736925f22821d35f","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T10:30:51Z\n Ended: 2026-09-22T10:46:49Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j5854208-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j5854208-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"c27576440b56d6d4939511cc79304412ff3db2952e23334938949fc5f7931800","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102324260725854208","occurred_at":"2026-09-22T09:08:57Z","failed_at":"alert","signature_id":"c04ac85d28af75dbac28d615edf81c669fe6d30a818360aeab15677cb35e9184","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T11:02:38Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j5854208-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-vdftx\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-vdftx (namespace arobit) on cluster ci01-j5854208-svc has averaged 126.8% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102322675463491584","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6945,"pr_state":"closed","pr_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","final_merged_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-22T09:02:39Z","started_at":"2026-09-22T09:02:39Z","completed_at":"2026-09-22T11:23:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T11:39:14.674512284Z","timing_metadata_checked_at":"2026-09-22T11:39:14.674512284Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T11:39:14.677204974Z","region_metadata_checked_at":"2026-09-22T11:39:14.677204974Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102322643775524864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6128,"pr_state":"open","pr_sha":"170f76835cc1a848764bf285f1e459b293c9fc67","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T09:02:32Z","started_at":"2026-09-22T09:02:32Z","completed_at":"2026-09-22T11:20:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T11:39:14.645467791Z","timing_metadata_checked_at":"2026-09-22T11:39:14.645467791Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T11:39:14.652018467Z","region_metadata_checked_at":"2026-09-22T11:39:14.652018467Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"8c3b25f538a09726093e2ff02d3d7cd02f1776643a221c02948ff8863f47195f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102322643775524864","occurred_at":"2026-09-22T09:02:32Z","failed_at":"e2e","signature_id":"cf31419f2da35385da91c35d1264050fd9b33627fcc4d2faa1e1ec321e4a403d","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.33\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013bb900\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-v799gb\" in resourcegroup=\"rg-np-version-upgrade-v799gb-zmkrwb8zmjvk\" to finish creating: GET https://rp.j5524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28ed3a88-9cec-48ac-aeee-bbf9bd78e17f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28ed3a88-9cec-48ac-aeee-bbf9bd78e17f\",\n \"name\": \"28ed3a88-9cec-48ac-aeee-bbf9bd78e17f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T10:16:17.959988536Z\",\n \"endTime\": \"2026-09-22T10:35:18.744539107Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.33\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013bb900\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-v799gb\" in resourcegroup=\"rg-np-version-upgrade-v799gb-zmkrwb8zmjvk\" to finish creating: GET https://rp.j5524864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28ed3a88-9cec-48ac-aeee-bbf9bd78e17f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28ed3a88-9cec-48ac-aeee-bbf9bd78e17f\",\n \"name\": \"28ed3a88-9cec-48ac-aeee-bbf9bd78e17f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T10:16:17.959988536Z\",\n \"endTime\": \"2026-09-22T10:35:18.744539107Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102322611546492928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6756,"pr_state":"open","pr_sha":"a65a65d0f5af3365546d2f648ce8be93984b79b0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T09:02:24Z","started_at":"2026-09-22T09:02:24Z","completed_at":"2026-09-22T11:44:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T12:00:42.487387562Z","timing_metadata_checked_at":"2026-09-22T12:00:42.487387562Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T12:00:42.493136043Z","region_metadata_checked_at":"2026-09-22T12:00:42.493136043Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"5fb9d695be97d0f965ece222e298e6fd45756655c9139b1da546e9ef8f80fdc6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102322611546492928","occurred_at":"2026-09-22T09:02:24Z","failed_at":"e2e","signature_id":"b72d73aed22f5ea9b8a850bcf6cbcbb6a47fcdcfbb032249b5650e114d6660b6","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001b460a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-9snnbjlt9gmm\" to finish deleting: GET https://rp.j6492928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c058261-e695-445d-9278-032944e20374\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c058261-e695-445d-9278-032944e20374\",\n \"name\": \"6c058261-e695-445d-9278-032944e20374\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T11:00:25.615396611Z\",\n \"endTime\": \"2026-09-22T11:24:30.827375508Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t00an45ebhqqrvdh3i5r6elagrf94qf still exists (deletion dispatched at 2026-09-22T11:00:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001b460a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-9snnbjlt9gmm\" to finish deleting: GET https://rp.j6492928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c058261-e695-445d-9278-032944e20374\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c058261-e695-445d-9278-032944e20374\",\n \"name\": \"6c058261-e695-445d-9278-032944e20374\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T11:00:25.615396611Z\",\n \"endTime\": \"2026-09-22T11:24:30.827375508Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t00an45ebhqqrvdh3i5r6elagrf94qf still exists (deletion dispatched at 2026-09-22T11:00:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d7e6f5f801a7d70310cb591ef7537ff80dca981f708bf906609e533efd30c303","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102322611546492928","occurred_at":"2026-09-22T09:02:24Z","failed_at":"alert","signature_id":"0eeb5b4008847244d310d0b89d87d8574f8d15776d3de636dae9d610aa73bc6b","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T11:09:39Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j6492928-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-7qnw7\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-7qnw7 (namespace arobit) on cluster ci01-j6492928-svc has averaged 126.5% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102322606454607872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T09:02:23Z","started_at":"2026-09-22T09:02:23Z","completed_at":"2026-09-22T11:37:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T11:50:03.124344484Z","timing_metadata_checked_at":"2026-09-22T11:50:03.124344484Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T11:50:03.12843307Z","region_metadata_checked_at":"2026-09-22T11:50:03.12843307Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"4e68a8248c50362b2c32928a134b7c4da51445df54a141602e0e6863fff3f9f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102322606454607872","occurred_at":"2026-09-22T09:02:23Z","failed_at":"alert","signature_id":"ad66a2666c34b923be42f086c71ddfb6acbad8f1b4774fa7a1f536a33d881b25","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T10:47:24Z\n Ended: 2026-09-22T11:06:27Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j4607872-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} exceeded 1 second over the past 30 minutes (cluster ci01-j4607872-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}},{"row_id":"5749dd7bd317324d45e6970036c930873076e6b65440ee57b97947c929e363fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102322606454607872","occurred_at":"2026-09-22T09:02:23Z","failed_at":"alert","signature_id":"54194dd46ed1dcbebcce9a1d3bc483a9cee37d6fb298a0432bda317c79de57b3","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T10:40:24Z\n Ended: 2026-09-22T10:46:25Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4607872-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4607872-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"c6453ebea696f464bac9dc1e4ec71e43b768bb9b25b28c4d4779321ae4f1a6e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102322606454607872","occurred_at":"2026-09-22T09:02:23Z","failed_at":"e2e","signature_id":"e67baa0dd3ecf57ed16b62b6102e97d4b4b5f1d7f193c304c25d15049f8d2d1d","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001600258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-6qgtplwqgwxc\" to finish deleting: GET https://rp.j4607872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7729f3c0-d445-4340-940f-e4e18c7a926c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7729f3c0-d445-4340-940f-e4e18c7a926c\",\n \"name\": \"7729f3c0-d445-4340-940f-e4e18c7a926c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T10:52:23.455866421Z\",\n \"endTime\": \"2026-09-22T11:16:32.851282845Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t008v8tdek58eks5u6a3h20d448qg5d still exists (deletion dispatched at 2026-09-22T10:52:24Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001600258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-6qgtplwqgwxc\" to finish deleting: GET https://rp.j4607872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7729f3c0-d445-4340-940f-e4e18c7a926c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7729f3c0-d445-4340-940f-e4e18c7a926c\",\n \"name\": \"7729f3c0-d445-4340-940f-e4e18c7a926c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T10:52:23.455866421Z\",\n \"endTime\": \"2026-09-22T11:16:32.851282845Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t008v8tdek58eks5u6a3h20d448qg5d still exists (deletion dispatched at 2026-09-22T10:52:24Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102322580831604736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6995,"pr_state":"closed","pr_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","final_merged_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T09:02:16Z","started_at":"2026-09-22T09:02:16Z","completed_at":"2026-09-22T11:25:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T11:39:14.69439571Z","timing_metadata_checked_at":"2026-09-22T11:39:14.69439571Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T11:39:14.698203196Z","region_metadata_checked_at":"2026-09-22T11:39:14.698203196Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"10ce1f20676968317a259c10209e739c9b0a125241dc7f9a8d8bdaa72b64aba9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102322580831604736","occurred_at":"2026-09-22T09:02:16Z","failed_at":"alert","signature_id":"511274e848bdfa86676025b44ef9d040a781c41c25c3a64f18cffbd4ffd6791d","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T11:08:02Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j1604736-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-b55rr\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-b55rr (namespace arobit) on cluster ci01-j1604736-svc has averaged 125.8% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"4c7e1593a3124b10e61dd09412a62e50515b917a891793dd4cad006cdc8df7d3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102322580831604736","occurred_at":"2026-09-22T09:02:16Z","failed_at":"e2e","signature_id":"68971a5f68fb618afb67d5665d57f00e7f0ad4c7b3d52339c1dca509cf1867fd","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a22108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-xvl2sk9668pf\" to finish deleting: GET https://rp.j1604736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/092084d5-d703-4f92-86f4-eb6914a0f751\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/092084d5-d703-4f92-86f4-eb6914a0f751\",\n \"name\": \"092084d5-d703-4f92-86f4-eb6914a0f751\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T10:43:39.247737464Z\",\n \"endTime\": \"2026-09-22T11:07:45.926705915Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t0064f6bcmvlot3qp483fs9rj85leb0 still exists (deletion dispatched at 2026-09-22T10:43:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a22108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-xvl2sk9668pf\" to finish deleting: GET https://rp.j1604736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/092084d5-d703-4f92-86f4-eb6914a0f751\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/092084d5-d703-4f92-86f4-eb6914a0f751\",\n \"name\": \"092084d5-d703-4f92-86f4-eb6914a0f751\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T10:43:39.247737464Z\",\n \"endTime\": \"2026-09-22T11:07:45.926705915Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t0064f6bcmvlot3qp483fs9rj85leb0 still exists (deletion dispatched at 2026-09-22T10:43:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7e41819a6fb7878dd1914f07c03adcf58738f3b39140e67f55af9b38dfb426dd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102322580831604736","occurred_at":"2026-09-22T09:02:16Z","failed_at":"alert","signature_id":"4b78bc370e58022f6fad2989fb6bd5f3829134afce729259ce754ba08b7ed2fc","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T10:32:45Z\n Ended: 2026-09-22T10:38:46Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1604736-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1604736-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6516/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102321677739233280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6516,"pr_state":"closed","pr_sha":"571559aeb25fe3e03ab5b299d1b520ff8d3206ca","final_merged_sha":"571559aeb25fe3e03ab5b299d1b520ff8d3206ca","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T08:58:41Z","started_at":"2026-09-22T08:58:41Z","completed_at":"2026-09-22T11:15:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T11:28:40.37882381Z","timing_metadata_checked_at":"2026-09-22T11:28:40.37882381Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T11:28:40.384375793Z","region_metadata_checked_at":"2026-09-22T11:28:40.384375793Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"207ee4e167c85b67626edfcb161c74afd8f86c687bda0ddd5555c502162711bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6516/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102321677739233280","occurred_at":"2026-09-22T08:58:41Z","failed_at":"alert","signature_id":"d87de921cb349c4db01e354fe18d2469501ad7fab0e50dc1fe219f4ec2f384c1","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T10:44:39Z\n Ended: 2026-09-22T10:56:41Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j9233280-svc\", component=\"frontend\", method=\"get\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j9233280-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}},{"row_id":"60f84579aa04615d18d0024d1abceff07827b57a905694dc9f2ecd524f547742","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6516/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102321677739233280","occurred_at":"2026-09-22T08:58:41Z","failed_at":"e2e","signature_id":"fac8a1be8be2e9eccb68207302d9e3c8f8c385ae84412c13899620b63793cfe6","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010ac9c0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-smq89jgn4gq5\" to finish creating: GET https://rp.j9233280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/37dd9b79-8013-4222-b75a-587194eb2514\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/37dd9b79-8013-4222-b75a-587194eb2514\",\n \"name\": \"37dd9b79-8013-4222-b75a-587194eb2514\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T10:03:21.992748868Z\",\n \"endTime\": \"2026-09-22T10:32:22.669673154Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010ac9c0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-smq89jgn4gq5\" to finish creating: GET https://rp.j9233280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/37dd9b79-8013-4222-b75a-587194eb2514\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/37dd9b79-8013-4222-b75a-587194eb2514\",\n \"name\": \"37dd9b79-8013-4222-b75a-587194eb2514\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T10:03:21.992748868Z\",\n \"endTime\": \"2026-09-22T10:32:22.669673154Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8200be30ee641e1bc0072d653033914461c1b27a31c7ea5acc52d568d24dbef9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6516/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102321677739233280","occurred_at":"2026-09-22T08:58:41Z","failed_at":"alert","signature_id":"c8a39895ab05da242fab6d0c9aa12843979da1c341ad1c2d1621b2491d4b33cd","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T10:24:38Z\n Ended: 2026-09-22T10:53:38Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j9233280-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.199:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9233280/providers/microsoft.monitor/accounts/hcps-j9233280\", namespace=\"ocm-arohcpci01-2t008r4997p9pdhbin6riaqnkh7p7od9-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-q8nsw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t008r4997p9pdhbin6riaqnkh7p7od9-v20260901/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-22T10:24:38Z\n Ended: 2026-09-22T10:50:38Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j9233280-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.199:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9233280/providers/microsoft.monitor/accounts/hcps-j9233280\", namespace=\"ocm-arohcpci01-2t008r4997p9pdhbin6riaqnkh7p7od9-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-q8nsw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t008r4997p9pdhbin6riaqnkh7p7od9-v20260901/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"ba1ba53b93db50970807dc2a36bf35976699a51c8349eb83d0b911527b89d7d2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6516/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102321677739233280","occurred_at":"2026-09-22T08:58:41Z","failed_at":"alert","signature_id":"9846f4c1be6a2dfdf4774339e5a3e0d83657d7196755eebca0d28ea4bac15bd9","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T10:39:33Z\n Ended: 2026-09-22T10:52:34Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9233280-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.199:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9233280/providers/microsoft.monitor/accounts/hcps-j9233280\", namespace=\"ocm-arohcpci01-2t008r4997p9pdhbin6riaqnkh7p7od9-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-q8nsw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t008r4997p9pdhbin6riaqnkh7p7od9-v20260901/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-22T10:49:32Z\n Ended: 2026-09-22T10:56:35Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9233280-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.199:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9233280/providers/microsoft.monitor/accounts/hcps-j9233280\", namespace=\"ocm-arohcpci01-2t008r4997p9pdhbin6riaqnkh7p7od9-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-q8nsw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t008r4997p9pdhbin6riaqnkh7p7od9-v20260901/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102319161173610496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5724,"pr_state":"closed","pr_sha":"02f55855ad2e600a742ce1b551002c7fbd872350","final_merged_sha":"02f55855ad2e600a742ce1b551002c7fbd872350","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-22T08:48:41Z","started_at":"2026-09-22T08:48:41Z","completed_at":"2026-09-22T10:52:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T11:07:46.756802219Z","timing_metadata_checked_at":"2026-09-22T11:07:46.756802219Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T11:07:46.760065108Z","region_metadata_checked_at":"2026-09-22T11:07:46.760065108Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102316023234433024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6756,"pr_state":"open","pr_sha":"a65a65d0f5af3365546d2f648ce8be93984b79b0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T08:36:13Z","started_at":"2026-09-22T08:36:13Z","completed_at":"2026-09-22T08:48:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T09:00:17.453735145Z","timing_metadata_checked_at":"2026-09-22T09:00:17.453735145Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-22T09:00:17.456916134Z","region_metadata_checked_at":"2026-09-22T10:04:14.30389561Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"0ee1f65c5159b0eae7032784c85512397810af9f4337c3b29a03af34a4b2c3de","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102316023234433024","occurred_at":"2026-09-22T08:36:13Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102307266211352576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6922,"pr_state":"open","pr_sha":"87e40a07142a1457ed3f83374cc5d085c87d8f29","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T08:00:53Z","started_at":"2026-09-22T08:00:53Z","completed_at":"2026-09-22T10:17:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T10:36:28.586034144Z","timing_metadata_checked_at":"2026-09-22T10:36:28.586034144Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T10:36:28.589538333Z","region_metadata_checked_at":"2026-09-22T10:36:28.589538333Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"9a795c26db1bb851e9242b300a8260682051e0f8c3f55848f729b648530af64e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102307266211352576","occurred_at":"2026-09-22T08:00:53Z","failed_at":"alert","signature_id":"006b30f9681fe7ccddd410e35778871e243e77c2a5c3d3f8a93cc41e1388ad31","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T09:16:29Z\n Ended: 2026-09-22T09:43:29Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1352576-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1352576-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"bf421805d44e87a6e587ee29ba0e3ee0f9bc09ad1b5abf82f7968f0b35eaa2ab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102307266211352576","occurred_at":"2026-09-22T08:00:53Z","failed_at":"e2e","signature_id":"57f6ec7707b46ec3fc5d943d1acba1050e14974e279203d5fb4268374fcc3f88","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a24150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-8cpg6p\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-9jjn6d27plq8\" to finish deleting: GET https://rp.j1352576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ba57e58c-f8d4-4d4b-904a-f3e851086567\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ba57e58c-f8d4-4d4b-904a-f3e851086567\",\n \"name\": \"ba57e58c-f8d4-4d4b-904a-f3e851086567\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T09:32:28.672968194Z\",\n \"endTime\": \"2026-09-22T09:56:36.170444319Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svvbd6lttd70rvgqctu642sbsj2vlqt still exists (deletion dispatched at 2026-09-22T09:32:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a24150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-8cpg6p\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-9jjn6d27plq8\" to finish deleting: GET https://rp.j1352576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ba57e58c-f8d4-4d4b-904a-f3e851086567\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ba57e58c-f8d4-4d4b-904a-f3e851086567\",\n \"name\": \"ba57e58c-f8d4-4d4b-904a-f3e851086567\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T09:32:28.672968194Z\",\n \"endTime\": \"2026-09-22T09:56:36.170444319Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svvbd6lttd70rvgqctu642sbsj2vlqt still exists (deletion dispatched at 2026-09-22T09:32:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102295446469742592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6503,"pr_state":"closed","pr_sha":"6c5cfd09b9bbf1585b1cd37220da16677f327452","final_merged_sha":"a0bf660bfdf32222374a82406e0907cae4b53869","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T07:14:27Z","started_at":"2026-09-22T07:14:27Z","completed_at":"2026-09-22T09:19:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T09:32:07.452138486Z","timing_metadata_checked_at":"2026-09-22T09:32:07.452138486Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T09:32:07.455708575Z","region_metadata_checked_at":"2026-09-22T09:32:07.455708575Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"27cefdd8a11d35018abff311986f522b753c3426eac5792036bf406e063ce1ff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102295446469742592","occurred_at":"2026-09-22T07:14:27Z","failed_at":"alert","signature_id":"e0e17143e5d3902f1f452df7ad652fc0f85ccf61ea0d86668034f00a58d0fea8","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T08:40:36Z\n Ended: 2026-09-22T08:56:35Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j9742592-svc\", component=\"frontend\", method=\"get\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters exceeded 1 second over the past 30 minutes (cluster ci01-j9742592-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}},{"row_id":"a81d0f6f555c7652781f743efa6a06507eee6d475cdffcafa59594a8d38be1b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102295446469742592","occurred_at":"2026-09-22T07:14:27Z","failed_at":"e2e","signature_id":"1eddc05aa88f49be251be96845a2a982e308ac66e6019a984d6ae05ad8c0e7a6","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00295c090\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-k8jcmpkl6wgx\" to finish deleting: GET https://rp.j9742592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2ccd7ef7-ba19-4730-b390-c22a5cb16ed8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2ccd7ef7-ba19-4730-b390-c22a5cb16ed8\",\n \"name\": \"2ccd7ef7-ba19-4730-b390-c22a5cb16ed8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T08:30:31.706415617Z\",\n \"endTime\": \"2026-09-22T08:54:37.036482039Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svughrdal726o55rr99lgebhg86q31q still exists (deletion dispatched at 2026-09-22T08:30:33Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00295c090\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-k8jcmpkl6wgx\" to finish deleting: GET https://rp.j9742592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2ccd7ef7-ba19-4730-b390-c22a5cb16ed8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2ccd7ef7-ba19-4730-b390-c22a5cb16ed8\",\n \"name\": \"2ccd7ef7-ba19-4730-b390-c22a5cb16ed8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T08:30:31.706415617Z\",\n \"endTime\": \"2026-09-22T08:54:37.036482039Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svughrdal726o55rr99lgebhg86q31q still exists (deletion dispatched at 2026-09-22T08:30:33Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ed186e3b89c1ab19b93b47492215ee209680852cca5e89b98e27506c384bd143","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102295446469742592","occurred_at":"2026-09-22T07:14:27Z","failed_at":"alert","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T08:16:37Z\n Ended: 2026-09-22T08:34:37Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9742592-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102294129609281536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"7cf0a28b3379c46112996aea37d6ac4e12d00ece","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T07:09:13Z","started_at":"2026-09-22T07:09:13Z","completed_at":"2026-09-22T09:31:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T09:42:40.485833366Z","timing_metadata_checked_at":"2026-09-22T09:42:40.485833366Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T09:42:40.489576653Z","region_metadata_checked_at":"2026-09-22T09:42:40.489576653Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"1d3b09a64d86ec29cc133e5374814b507fb979a3f9ea6678b37d5639d8d25663","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102294129609281536","occurred_at":"2026-09-22T07:09:13Z","failed_at":"e2e","signature_id":"e8f2d1e264229b040ff7f19d77ed2f91221abe1ec550da844d4003af2d5756a9","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:115]: failed to verify simple web app runs on cluster \"cluster-hs-pre-84mf8p\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001186040\u003e: \n DNS for route host agnhost-e2e-sample-app-zjd8b.apps.aro.g7i1i4o8r4c6r5x.5s62.j9281536.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-zjd8b.apps.aro.g7i1i4o8r4c6r5x.5s62.j9281536.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-zjd8b.apps.aro.g7i1i4o8r4c6r5x.5s62.j9281536.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:115]: failed to verify simple web app runs on cluster \"cluster-hs-pre-84mf8p\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001186040\u003e: \n DNS for route host agnhost-e2e-sample-app-zjd8b.apps.aro.g7i1i4o8r4c6r5x.5s62.j9281536.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-zjd8b.apps.aro.g7i1i4o8r4c6r5x.5s62.j9281536.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-zjd8b.apps.aro.g7i1i4o8r4c6r5x.5s62.j9281536.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-wrj85.apps.aro.h6n7n8r8p3m8n7w.1qn2.j8935808.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-wrj85.apps.aro.h6n7n8r8p3m8n7w.1qn2.j8935808.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wrj85.apps.aro.h6n7n8r8p3m8n7w.1qn2.j8935808.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded"}},{"row_id":"51ee070c4c93d196139a1b750137ac0484054921b9d992fa39f8aea4917a115c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102294129609281536","occurred_at":"2026-09-22T07:09:13Z","failed_at":"e2e","signature_id":"e6d21a6a368ee63de4c6cdddfa3fe07749c51b83e183aac5c0c8f26cff689fee","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc0012181b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-7dpf6w54wkcq\" to finish deleting: GET https://rp.j9281536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b5834841-df9c-4967-9c2d-7ef56f8c94f8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b5834841-df9c-4967-9c2d-7ef56f8c94f8\",\n \"name\": \"b5834841-df9c-4967-9c2d-7ef56f8c94f8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T08:48:33.543198293Z\",\n \"endTime\": \"2026-09-22T09:12:40.623611763Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svudbr2hssobmqj9amh0pf72td9kpdk still exists (deletion dispatched at 2026-09-22T08:48:35Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc0012181b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-7dpf6w54wkcq\" to finish deleting: GET https://rp.j9281536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b5834841-df9c-4967-9c2d-7ef56f8c94f8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b5834841-df9c-4967-9c2d-7ef56f8c94f8\",\n \"name\": \"b5834841-df9c-4967-9c2d-7ef56f8c94f8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T08:48:33.543198293Z\",\n \"endTime\": \"2026-09-22T09:12:40.623611763Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svudbr2hssobmqj9amh0pf72td9kpdk still exists (deletion dispatched at 2026-09-22T08:48:35Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"65256360f523e30df3885e893462613a9150513edcafe1228c6c964ced6c5a73","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102294129609281536","occurred_at":"2026-09-22T07:09:13Z","failed_at":"e2e","signature_id":"4d3127d69b3237c6f0f072b82641079832ba20fe44b9241738bc8153a3a12cbc","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc001bd41e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-xlzps9\" in resourcegroup=\"rg-np-version-upgrade-xlzps9-5jp45ggx94q5\" to finish deleting: GET https://rp.j9281536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bab02b80-e2c9-493c-8702-48bdacd5cfbd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bab02b80-e2c9-493c-8702-48bdacd5cfbd\",\n \"name\": \"bab02b80-e2c9-493c-8702-48bdacd5cfbd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T08:41:40.162628083Z\",\n \"endTime\": \"2026-09-22T09:05:40.589950549Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svudcl0k8f6n7hat0l3mq2kc5oo9gbd still exists (deletion dispatched at 2026-09-22T08:41:42Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc001bd41e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-xlzps9\" in resourcegroup=\"rg-np-version-upgrade-xlzps9-5jp45ggx94q5\" to finish deleting: GET https://rp.j9281536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bab02b80-e2c9-493c-8702-48bdacd5cfbd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bab02b80-e2c9-493c-8702-48bdacd5cfbd\",\n \"name\": \"bab02b80-e2c9-493c-8702-48bdacd5cfbd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T08:41:40.162628083Z\",\n \"endTime\": \"2026-09-22T09:05:40.589950549Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svudcl0k8f6n7hat0l3mq2kc5oo9gbd still exists (deletion dispatched at 2026-09-22T08:41:42Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7470d259406a65035c7d84489510508ee9139681885c6be24558b1921f8b7df9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102294129609281536","occurred_at":"2026-09-22T07:09:13Z","failed_at":"e2e","signature_id":"eec6e687779bd9082c44d49d3a20813b73f9a8c303884772641366c4bd373d06","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc0015100c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-9sdm78\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-sb4lq8xcf9k6\" to finish deleting: GET https://rp.j9281536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a7133fbd-4d17-4f3b-aa59-ab3b1ce58d8a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a7133fbd-4d17-4f3b-aa59-ab3b1ce58d8a\",\n \"name\": \"a7133fbd-4d17-4f3b-aa59-ab3b1ce58d8a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T08:30:59.74305005Z\",\n \"endTime\": \"2026-09-22T08:55:00.567673289Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svudf4qkt2oamjltcf461o8t5os6if5 still exists (deletion dispatched at 2026-09-22T08:31:00Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc0015100c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-9sdm78\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-sb4lq8xcf9k6\" to finish deleting: GET https://rp.j9281536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a7133fbd-4d17-4f3b-aa59-ab3b1ce58d8a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a7133fbd-4d17-4f3b-aa59-ab3b1ce58d8a\",\n \"name\": \"a7133fbd-4d17-4f3b-aa59-ab3b1ce58d8a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T08:30:59.74305005Z\",\n \"endTime\": \"2026-09-22T08:55:00.567673289Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svudf4qkt2oamjltcf461o8t5os6if5 still exists (deletion dispatched at 2026-09-22T08:31:00Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102292246979153920","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6869,"pr_state":"open","pr_sha":"45619559b0273524877730f5066212561774a32d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T07:01:28Z","started_at":"2026-09-22T07:01:28Z","completed_at":"2026-09-22T08:10:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T08:19:00.0916377Z","timing_metadata_checked_at":"2026-09-22T08:19:00.0916377Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T08:19:00.095472288Z","region_metadata_checked_at":"2026-09-22T08:19:00.095472288Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"3628b34b58fda532874eeddf07dbdbcc36b5f942f3c0d76b930f4ad986b11c40","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102292246979153920","occurred_at":"2026-09-22T07:01:28Z","failed_at":"provision","signature_id":"de77c358890614e2ec342728c508f5e003c54336cd8a63368d4ae3d9e2e1bb39","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-22T07:33:19.014Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-09-22T08:03:18.587Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-22T07:33:19.023908785Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j9153920-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j9153920-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\"}\\n{\\\"time\\\":\\\"2026-09-22T07:33:21.03768985Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j9153920-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j9153920-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-22T07:33:21.167202045Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j9153920-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j9153920-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1f0d1c0d9bb7c84ece352b5ff00277205bc7b31c723e9e6b1adc5bdf14ad3d00","failure_pattern":"error running Shell Step, failed to execute shell command: {\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"\u003credacted-id\u003e\",\"resourceGroup\":\"hcp-underlay-ci01-j9153920-mgmt-2\",\"clusterName\":\"ci01-j915392","search_query":"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-22T07:33:19.023908785Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j9153920-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j9153920-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\"}\\n{\\\"time\\\":\\\"2026-09-22T07:33:21.03768985Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j9153920-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j9153920-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-22T07:33:21.167202045Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j9153920-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j9153920-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1f0d1c0d9bb7c84ece352b5ff00277205bc7b31c723e9e6b1adc5bdf14ad3d00"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7060/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102289975184723968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7060,"pr_state":"closed","pr_sha":"b80303ca0c8d1a96771675fae2b83405b05c8407","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T06:52:43Z","started_at":"2026-09-22T06:52:43Z","completed_at":"2026-09-22T07:59:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T08:08:54.760885843Z","timing_metadata_checked_at":"2026-09-22T08:08:54.760885843Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T08:08:54.764792131Z","region_metadata_checked_at":"2026-09-22T08:08:54.764792131Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"db9f3f1be471ae914a0fb4564373eb0a5b1f6423c35d578676e6f82bc73883c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7060/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102289975184723968","occurred_at":"2026-09-22T06:52:43Z","failed_at":"provision","signature_id":"546713f253dbb07bdf6d6b240d468c73b1927023c4bbf21dee04e87aaa304e9a","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"templatize-pipeline","failure_text":"time=2026-09-22T07:21:38.848Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-09-22T07:21:40.354Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-09-22T07:21:41.548Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=d03eff9eca8ce6668b3735feb0fed36048a18f0177222fc337f65cc4e325d887 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4723968-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fd03eff9eca8ce6668b3735feb0fed36048a18f0177222fc337f65cc4e325d887\ntime=2026-09-22T07:51:38.563Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102288391285510144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6128,"pr_state":"open","pr_sha":"170f76835cc1a848764bf285f1e459b293c9fc67","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T06:46:25Z","started_at":"2026-09-22T06:46:25Z","completed_at":"2026-09-22T09:00:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T09:10:35.118450465Z","timing_metadata_checked_at":"2026-09-22T09:10:35.118450465Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T09:10:35.122319153Z","region_metadata_checked_at":"2026-09-22T09:10:35.122319153Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b25690f7187e26def47631919f86b40e9989fb9ce1c96468fd19d7a5b9e88f3e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102288391285510144","occurred_at":"2026-09-22T06:46:25Z","failed_at":"e2e","signature_id":"0cdd11030bef97770abbc84eb63e790e0a2d96f7178f49901404981b327067e2","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001857b30\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-5-0-9bf8n6\" in resourcegroup=\"rg-zstream-upgrade-5-0-nkz6rtcklccm\" to finish deleting: GET https://rp.j5510144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fad47a2e-73a1-4972-a714-79b849fba16a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fad47a2e-73a1-4972-a714-79b849fba16a\",\n \"name\": \"fad47a2e-73a1-4972-a714-79b849fba16a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T08:11:14.087337109Z\",\n \"endTime\": \"2026-09-22T08:35:22.583136314Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svu5bos2fu9mlb7h90mr26252ojhi8d still exists (deletion dispatched at 2026-09-22T08:11:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001857b30\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-5-0-9bf8n6\" in resourcegroup=\"rg-zstream-upgrade-5-0-nkz6rtcklccm\" to finish deleting: GET https://rp.j5510144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fad47a2e-73a1-4972-a714-79b849fba16a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fad47a2e-73a1-4972-a714-79b849fba16a\",\n \"name\": \"fad47a2e-73a1-4972-a714-79b849fba16a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T08:11:14.087337109Z\",\n \"endTime\": \"2026-09-22T08:35:22.583136314Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svu5bos2fu9mlb7h90mr26252ojhi8d still exists (deletion dispatched at 2026-09-22T08:11:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7066/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102288344015704064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7066,"pr_state":"closed","pr_sha":"eca364775bafae8fcd8399b7d0db91dc6fb25c6d","final_merged_sha":"23179a5354aa06a57efca1f82118702c3e475ce6","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T06:46:14Z","started_at":"2026-09-22T06:46:14Z","completed_at":"2026-09-22T08:57:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T09:10:35.787224149Z","timing_metadata_checked_at":"2026-09-22T09:10:35.787224149Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T09:10:35.789883041Z","region_metadata_checked_at":"2026-09-22T09:10:35.789883041Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"3fff2be1ef58c3cf0ce924c95fe5baf9cf163ceabaa33fd836bd064f55208bb7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7066/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102288344015704064","occurred_at":"2026-09-22T06:46:14Z","failed_at":"alert","signature_id":"8b95f05c94191a82a105dd423957340c94a99c1f3d6ef1c8917fd69f25e7d61e","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T08:02:38Z\n Ended: 2026-09-22T08:15:39Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j5704064-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j5704064-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"a66b77bca658dee9f00706bf8fdd21d1e2826b18a6f509eb4a2038fc5982aa09","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7066/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102288344015704064","occurred_at":"2026-09-22T06:46:14Z","failed_at":"e2e","signature_id":"47ebdfd44bc097708d001bb4432cee3e0d62f312c88e513f44ba0350a0c2b83e","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000848ab0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-2hzskj2pr2m2\" to finish deleting: GET https://rp.j5704064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b8310f47-3419-4b91-8a4e-4790ddca0cbb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b8310f47-3419-4b91-8a4e-4790ddca0cbb\",\n \"name\": \"b8310f47-3419-4b91-8a4e-4790ddca0cbb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T08:12:16.220105938Z\",\n \"endTime\": \"2026-09-22T08:36:24.406787177Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svu302f6dr3da93kcsml9jev780gduh still exists (deletion dispatched at 2026-09-22T08:12:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000848ab0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-2hzskj2pr2m2\" to finish deleting: GET https://rp.j5704064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b8310f47-3419-4b91-8a4e-4790ddca0cbb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b8310f47-3419-4b91-8a4e-4790ddca0cbb\",\n \"name\": \"b8310f47-3419-4b91-8a4e-4790ddca0cbb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T08:12:16.220105938Z\",\n \"endTime\": \"2026-09-22T08:36:24.406787177Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svu302f6dr3da93kcsml9jev780gduh still exists (deletion dispatched at 2026-09-22T08:12:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b6321381d21f75564aa30a55002568822c3e099fbe5d4142fb15cad868bbfb85","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7066/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102288344015704064","occurred_at":"2026-09-22T06:46:14Z","failed_at":"e2e","signature_id":"de54a79a3cfc986fa81033de7f4ed2ecec96374717c110bb677c9614805470bd","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000a665d0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-5ckwfmkg58dx\" to finish deleting: GET https://rp.j5704064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9c4a5f58-2281-43f4-9356-11220cd3316d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9c4a5f58-2281-43f4-9356-11220cd3316d\",\n \"name\": \"9c4a5f58-2281-43f4-9356-11220cd3316d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T08:01:34.997609657Z\",\n \"endTime\": \"2026-09-22T08:25:34.356075446Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svu3n2uvu3jjl757qfdq4ivet0ki4e5 still exists (deletion dispatched at 2026-09-22T08:01:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000a665d0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-5ckwfmkg58dx\" to finish deleting: GET https://rp.j5704064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9c4a5f58-2281-43f4-9356-11220cd3316d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9c4a5f58-2281-43f4-9356-11220cd3316d\",\n \"name\": \"9c4a5f58-2281-43f4-9356-11220cd3316d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T08:01:34.997609657Z\",\n \"endTime\": \"2026-09-22T08:25:34.356075446Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svu3n2uvu3jjl757qfdq4ivet0ki4e5 still exists (deletion dispatched at 2026-09-22T08:01:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102286205365260288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6995,"pr_state":"closed","pr_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","final_merged_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T06:37:44Z","started_at":"2026-09-22T06:37:44Z","completed_at":"2026-09-22T08:44:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T09:00:17.475735875Z","timing_metadata_checked_at":"2026-09-22T09:00:17.475735875Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T09:00:17.478456766Z","region_metadata_checked_at":"2026-09-22T09:00:17.478456766Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"64899ff5effd3e89925702f5df54ca7c5a16803748ab7ec27276844e42d85d27","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102286205365260288","occurred_at":"2026-09-22T06:37:44Z","failed_at":"e2e","signature_id":"5331077f0794dfd9d86a1d07f697ecc5bb7dd9ee6da2ea69bbcff92a887a5cbf","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011159a0\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-dxqcht\" in resourcegroup=\"rg-np-noedge-dxqcht-6zchcrtc2jzz\" to finish creating: GET https://rp.j5260288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9158be66-cd26-49d4-a60f-90293011d19d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9158be66-cd26-49d4-a60f-90293011d19d\",\n \"name\": \"9158be66-cd26-49d4-a60f-90293011d19d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T07:34:55.729794931Z\",\n \"endTime\": \"2026-09-22T07:53:56.208999403Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011159a0\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-dxqcht\" in resourcegroup=\"rg-np-noedge-dxqcht-6zchcrtc2jzz\" to finish creating: GET https://rp.j5260288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9158be66-cd26-49d4-a60f-90293011d19d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9158be66-cd26-49d4-a60f-90293011d19d\",\n \"name\": \"9158be66-cd26-49d4-a60f-90293011d19d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T07:34:55.729794931Z\",\n \"endTime\": \"2026-09-22T07:53:56.208999403Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a1e1833b5487005023324a913ac9bdaa501ad7407b9ec4a4b39785e5d91e6edf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102286205365260288","occurred_at":"2026-09-22T06:37:44Z","failed_at":"e2e","signature_id":"da19aec62f53374613dfa57e08b8efed775a8884eb19a2bb32427dac86c41c5c","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0008fe348\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-9c9tl7wq8vk4\" to finish deleting: GET https://rp.j5260288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fc7ae76f-bb09-4015-9244-5d0a185fd088\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fc7ae76f-bb09-4015-9244-5d0a185fd088\",\n \"name\": \"fc7ae76f-bb09-4015-9244-5d0a185fd088\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T07:58:38.947860134Z\",\n \"endTime\": \"2026-09-22T08:22:41.917812685Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svttdmiuctqkhavuc9550dhp1jpput2 still exists (deletion dispatched at 2026-09-22T07:58:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0008fe348\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-9c9tl7wq8vk4\" to finish deleting: GET https://rp.j5260288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fc7ae76f-bb09-4015-9244-5d0a185fd088\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fc7ae76f-bb09-4015-9244-5d0a185fd088\",\n \"name\": \"fc7ae76f-bb09-4015-9244-5d0a185fd088\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T07:58:38.947860134Z\",\n \"endTime\": \"2026-09-22T08:22:41.917812685Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svttdmiuctqkhavuc9550dhp1jpput2 still exists (deletion dispatched at 2026-09-22T07:58:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d3cd55ef2844d9bca26fb98a8de75a800dcd6e37c805cc4b0fed9037abc78ee9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102286205365260288","occurred_at":"2026-09-22T06:37:44Z","failed_at":"e2e","signature_id":"663948fad2f655a8f86778686b2675b46d2a717e3dc1afdecbbc520028b3d155","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-hb6mtzv8hm6q\" to finish creating: GET https://rp.j5260288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b4e55441-95a5-466c-adac-c574d8bcdb81\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b4e55441-95a5-466c-adac-c574d8bcdb81\",\n \"name\": \"b4e55441-95a5-466c-adac-c574d8bcdb81\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T07:34:05.103384159Z\",\n \"endTime\": \"2026-09-22T07:53:06.353871266Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n \u003c[]error | len:1, cap:1\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-hb6mtzv8hm6q\" to finish creating: GET https://rp.j5260288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b4e55441-95a5-466c-adac-c574d8bcdb81\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b4e55441-95a5-466c-adac-c574d8bcdb81\",\n \"name\": \"b4e55441-95a5-466c-adac-c574d8bcdb81\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T07:34:05.103384159Z\",\n \"endTime\": \"2026-09-22T07:53:06.353871266Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n \u003c[]error | len:1, cap:1\u003e: ...\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5","failure_pattern":"--------------------------------------------------------------------------------","search_query":"--------------------------------------------------------------------------------"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102283819666116608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7041,"pr_state":"open","pr_sha":"383b3840a43a95f99f5e04e35ba2b1a3a9ff774b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T06:28:15Z","started_at":"2026-09-22T06:28:15Z","completed_at":"2026-09-22T08:42:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T08:49:34.788923411Z","timing_metadata_checked_at":"2026-09-22T08:49:34.788923411Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T08:49:34.792089299Z","region_metadata_checked_at":"2026-09-22T08:49:34.792089299Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"1703d13aac7673602251aed91e17671b648a874e7ba81452f7ee51cc241862e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102283819666116608","occurred_at":"2026-09-22T06:28:15Z","failed_at":"e2e","signature_id":"fd972711a887999b78faa8f83122475bc265f258ed4abf1613070a868375c0e3","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0010d8438\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-dg-minor-gzv72w\" in resourcegroup=\"rg-np-dg-minor-gzv72w-qbl26k5f9hc6\" to finish deleting: GET https://rp.j6116608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/70b7b6e5-3879-4a3d-8927-7d4f3c24a792\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/70b7b6e5-3879-4a3d-8927-7d4f3c24a792\",\n \"name\": \"70b7b6e5-3879-4a3d-8927-7d4f3c24a792\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T08:00:43.888264873Z\",\n \"endTime\": \"2026-09-22T08:24:52.37976402Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svtqs2972vd4pkj0oea8s7g10qqc55p still exists (deletion dispatched at 2026-09-22T08:00:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0010d8438\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-dg-minor-gzv72w\" in resourcegroup=\"rg-np-dg-minor-gzv72w-qbl26k5f9hc6\" to finish deleting: GET https://rp.j6116608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/70b7b6e5-3879-4a3d-8927-7d4f3c24a792\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/70b7b6e5-3879-4a3d-8927-7d4f3c24a792\",\n \"name\": \"70b7b6e5-3879-4a3d-8927-7d4f3c24a792\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T08:00:43.888264873Z\",\n \"endTime\": \"2026-09-22T08:24:52.37976402Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svtqs2972vd4pkj0oea8s7g10qqc55p still exists (deletion dispatched at 2026-09-22T08:00:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"17bc280d0916080d169c75fbd26e9dd856d00bf0a4be0555c2a916590c629575","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102283819666116608","occurred_at":"2026-09-22T06:28:15Z","failed_at":"e2e","signature_id":"ae65cebd1e31dcea26a17fc935f9947f5012c586a63c60d1a7132f4c0742cf50","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001b00108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-m5blp6\" in resourcegroup=\"admin-credential-lifecycle-test-cdw82gf9mgkj\" to finish deleting: GET https://rp.j6116608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34fd4832-1a9c-4c2b-8b21-92a0603c3364\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34fd4832-1a9c-4c2b-8b21-92a0603c3364\",\n \"name\": \"34fd4832-1a9c-4c2b-8b21-92a0603c3364\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T07:41:21.159244981Z\",\n \"endTime\": \"2026-09-22T08:05:24.861502692Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svtqqq0lva06van6d9rveis55f6keot still exists (deletion dispatched at 2026-09-22T07:41:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001b00108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-m5blp6\" in resourcegroup=\"admin-credential-lifecycle-test-cdw82gf9mgkj\" to finish deleting: GET https://rp.j6116608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34fd4832-1a9c-4c2b-8b21-92a0603c3364\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34fd4832-1a9c-4c2b-8b21-92a0603c3364\",\n \"name\": \"34fd4832-1a9c-4c2b-8b21-92a0603c3364\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T07:41:21.159244981Z\",\n \"endTime\": \"2026-09-22T08:05:24.861502692Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svtqqq0lva06van6d9rveis55f6keot still exists (deletion dispatched at 2026-09-22T07:41:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4b984db07fccefa4e41a155e685263f92e8136a1000dffcfa0d513605929cf68","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102283819666116608","occurred_at":"2026-09-22T06:28:15Z","failed_at":"alert","signature_id":"450bab3ad58a1721f7009c09abe7807541a8c718e9c8576a60a01ca17fd9c0eb","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T08:31:25Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j6116608-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-zm7fl\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-zm7fl (namespace arobit) on cluster ci01-j6116608-svc has averaged 126.1% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"d9b4d9e173d2ed411888933b2545f6d390807f5c65cc7ff64418752cad155375","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102283819666116608","occurred_at":"2026-09-22T06:28:15Z","failed_at":"e2e","signature_id":"b12375b84d62728be721797e6df5f1f2636a84e36bca55635e5688467a22cefb","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001349428\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-4zk8nv92zkwr\" to finish creating: GET https://rp.j6116608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/325e6a8c-0d6c-4807-996c-1351e4e5c6e0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/325e6a8c-0d6c-4807-996c-1351e4e5c6e0\",\n \"name\": \"325e6a8c-0d6c-4807-996c-1351e4e5c6e0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T07:27:52.829680188Z\",\n \"endTime\": \"2026-09-22T07:46:55.075402091Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001349428\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-4zk8nv92zkwr\" to finish creating: GET https://rp.j6116608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/325e6a8c-0d6c-4807-996c-1351e4e5c6e0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/325e6a8c-0d6c-4807-996c-1351e4e5c6e0\",\n \"name\": \"325e6a8c-0d6c-4807-996c-1351e4e5c6e0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T07:27:52.829680188Z\",\n \"endTime\": \"2026-09-22T07:46:55.075402091Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102276754965008384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7053,"pr_state":"open","pr_sha":"1c995b8c43e3ab7b347b1bdb08b62da193b79171","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T06:00:11Z","started_at":"2026-09-22T06:00:11Z","completed_at":"2026-09-22T08:14:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T08:29:05.082713249Z","timing_metadata_checked_at":"2026-09-22T08:29:05.082713249Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T08:29:05.087701831Z","region_metadata_checked_at":"2026-09-22T08:29:05.087701831Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"114e7de8fb0df5024291edd61a0c5001db64bfbbb354ddbbad317ea50c7ad667","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102276754965008384","occurred_at":"2026-09-22T06:00:11Z","failed_at":"e2e","signature_id":"c7ab45585cce0b2cd1a21ce1fe6993c7311b39d4cfa42e2e851f63f05e62830d","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000a580f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-7hhc2zr8vjxs\" to finish deleting: GET https://rp.j5008384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a3241016-1253-4da6-8623-c49473e08ffb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a3241016-1253-4da6-8623-c49473e08ffb\",\n \"name\": \"a3241016-1253-4da6-8623-c49473e08ffb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T07:21:14.063772758Z\",\n \"endTime\": \"2026-09-22T07:45:20.292071261Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svtbp449eoj3qf5nbs7ctd9n6sit8tg still exists (deletion dispatched at 2026-09-22T07:21:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000a580f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-7hhc2zr8vjxs\" to finish deleting: GET https://rp.j5008384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a3241016-1253-4da6-8623-c49473e08ffb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a3241016-1253-4da6-8623-c49473e08ffb\",\n \"name\": \"a3241016-1253-4da6-8623-c49473e08ffb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T07:21:14.063772758Z\",\n \"endTime\": \"2026-09-22T07:45:20.292071261Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svtbp449eoj3qf5nbs7ctd9n6sit8tg still exists (deletion dispatched at 2026-09-22T07:21:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9081d8145a1431837436bb34d3b915134caeb7b61a986d5fc48d482404fa5960","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102276754965008384","occurred_at":"2026-09-22T06:00:11Z","failed_at":"e2e","signature_id":"75ba3b9e29e1bbce0b25702e1769f68adca116986257d7f8084087bb33628dd3","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:115]: failed to verify simple web app runs on cluster \"cluster-hs-pre-svhj5w\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017e4720\u003e: \n DNS for route host agnhost-e2e-sample-app-sxj6m.apps.aro.b5e0p3v1r0t3r4t.l2s4.j5008384.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-sxj6m.apps.aro.b5e0p3v1r0t3r4t.l2s4.j5008384.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sxj6m.apps.aro.b5e0p3v1r0t3r4t.l2s4.j5008384.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:115]: failed to verify simple web app runs on cluster \"cluster-hs-pre-svhj5w\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017e4720\u003e: \n DNS for route host agnhost-e2e-sample-app-sxj6m.apps.aro.b5e0p3v1r0t3r4t.l2s4.j5008384.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-sxj6m.apps.aro.b5e0p3v1r0t3r4t.l2s4.j5008384.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sxj6m.apps.aro.b5e0p3v1r0t3r4t.l2s4.j5008384.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-g6m95.apps.aro.v20261001.kkqb.j8719872.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-g6m95.apps.aro.v20261001.kkqb.j8719872.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g6m95.apps.aro.v20261001.kkqb.j8719872.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"d2533d02d597f5db47985bdeca4923eac4638f3b1c8087e5432011dc8b625008","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102276754965008384","occurred_at":"2026-09-22T06:00:11Z","failed_at":"e2e","signature_id":"428da8905e3f807461473c236e1fbd94195470e327e2f10f57443086600ef92c","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001bac198\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-2c7gjb8qcjb8\" to finish deleting: GET https://rp.j5008384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8a37ffb4-e28e-4493-a3e8-361ee859d5a8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8a37ffb4-e28e-4493-a3e8-361ee859d5a8\",\n \"name\": \"8a37ffb4-e28e-4493-a3e8-361ee859d5a8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T07:31:49.427817516Z\",\n \"endTime\": \"2026-09-22T07:55:50.330781697Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svtbhu4l0og0la7ciu7nisn164rc9se still exists (deletion dispatched at 2026-09-22T07:31:51Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001bac198\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-2c7gjb8qcjb8\" to finish deleting: GET https://rp.j5008384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8a37ffb4-e28e-4493-a3e8-361ee859d5a8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8a37ffb4-e28e-4493-a3e8-361ee859d5a8\",\n \"name\": \"8a37ffb4-e28e-4493-a3e8-361ee859d5a8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T07:31:49.427817516Z\",\n \"endTime\": \"2026-09-22T07:55:50.330781697Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svtbhu4l0og0la7ciu7nisn164rc9se still exists (deletion dispatched at 2026-09-22T07:31:51Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102276703479926784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5846,"pr_state":"closed","pr_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","final_merged_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T05:59:59Z","started_at":"2026-09-22T05:59:59Z","completed_at":"2026-09-22T08:12:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T08:29:04.941862165Z","timing_metadata_checked_at":"2026-09-22T08:29:04.941862165Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T08:29:04.945111053Z","region_metadata_checked_at":"2026-09-22T08:29:04.945111053Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"26e0dfcd686e2aa0a7f18eaeb11acd23ca2a49ed77b138fe43cb5945728ebafc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102276703479926784","occurred_at":"2026-09-22T05:59:59Z","failed_at":"e2e","signature_id":"7f12d0e1a62bedb37c8fe232dd318ca8e10bd4f5442c8d1539601e78b02c5186","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0019a42b8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-prv579\" in resourcegroup=\"rg-candidate-4-20-s4b6jx4lj9fq\" to finish deleting: GET https://rp.j9926784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a55d2b2a-4b2b-4856-9356-be804c0b554f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a55d2b2a-4b2b-4856-9356-be804c0b554f\",\n \"name\": \"a55d2b2a-4b2b-4856-9356-be804c0b554f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T07:21:32.842959151Z\",\n \"endTime\": \"2026-09-22T07:45:33.914148356Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svtfurgd77dtrpps49catdbplg5jon7 still exists (deletion dispatched at 2026-09-22T07:21:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0019a42b8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-prv579\" in resourcegroup=\"rg-candidate-4-20-s4b6jx4lj9fq\" to finish deleting: GET https://rp.j9926784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a55d2b2a-4b2b-4856-9356-be804c0b554f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a55d2b2a-4b2b-4856-9356-be804c0b554f\",\n \"name\": \"a55d2b2a-4b2b-4856-9356-be804c0b554f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T07:21:32.842959151Z\",\n \"endTime\": \"2026-09-22T07:45:33.914148356Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svtfurgd77dtrpps49catdbplg5jon7 still exists (deletion dispatched at 2026-09-22T07:21:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e673fa8d37df5592460fe01e0d00602d192fd363565a18f2e573af5e8378b82a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102276703479926784","occurred_at":"2026-09-22T05:59:59Z","failed_at":"e2e","signature_id":"620a806bdf69f493c7cd866ed882325652032b150b55c86a742a8b7ce5cc719b","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011c8090\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-4vwc4ffksj6w\" to finish deleting: GET https://rp.j9926784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1ab4f584-13dc-4529-b3a7-52bf12ff4323\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1ab4f584-13dc-4529-b3a7-52bf12ff4323\",\n \"name\": \"1ab4f584-13dc-4529-b3a7-52bf12ff4323\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T07:23:42.071372233Z\",\n \"endTime\": \"2026-09-22T07:47:43.944680192Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svth6c8pterthepvbj92trdph5hdiit still exists (deletion dispatched at 2026-09-22T07:23:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011c8090\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-4vwc4ffksj6w\" to finish deleting: GET https://rp.j9926784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1ab4f584-13dc-4529-b3a7-52bf12ff4323\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1ab4f584-13dc-4529-b3a7-52bf12ff4323\",\n \"name\": \"1ab4f584-13dc-4529-b3a7-52bf12ff4323\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T07:23:42.071372233Z\",\n \"endTime\": \"2026-09-22T07:47:43.944680192Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svth6c8pterthepvbj92trdph5hdiit still exists (deletion dispatched at 2026-09-22T07:23:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102276691408719872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T05:59:56Z","started_at":"2026-09-22T05:59:56Z","completed_at":"2026-09-22T08:09:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T08:19:00.301874334Z","timing_metadata_checked_at":"2026-09-22T08:19:00.301874334Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T08:19:00.304489126Z","region_metadata_checked_at":"2026-09-22T08:19:00.304489126Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"18d020450b626b7ee5ed842006931efa4cc2e4ab866860053cfa0884fbb05de4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102276691408719872","occurred_at":"2026-09-22T05:59:56Z","failed_at":"alert","signature_id":"a1415df35e18dcf241dab67e086a78b1cecb3327d794d9305875cc8e410a17fb","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T07:14:06Z\n Ended: 2026-09-22T07:47:06Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j8719872-svc\", component=\"frontend\", method=\"get\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} exceeded 1 second over the past 30 minutes (cluster ci01-j8719872-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}},{"row_id":"25b7ce1c096f900dff4cf7408eebcff9dda4f0c75c9a58e0ecada25d590feb34","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102276691408719872","occurred_at":"2026-09-22T05:59:56Z","failed_at":"alert","signature_id":"d0f8abdb9394d0098892a582f88ae35796d256dfa90792b902dd5844886e4f9b","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T07:13:07Z\n Ended: 2026-09-22T07:23:10Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j8719872-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j8719872-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"663a3faaef9d21449b6f2593b364cdef93a8aa6d27c77f616fdc7cb87ec30268","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102276691408719872","occurred_at":"2026-09-22T05:59:56Z","failed_at":"e2e","signature_id":"efe8eb11fb7eb1e937c8ca8f32c5d4c2908dd8dfebec4f5bc55c17c0ba957c3c","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:157]: failed to verify simple web app runs on v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001120bc0\u003e: \n DNS for route host agnhost-e2e-sample-app-g6m95.apps.aro.v20261001.kkqb.j8719872.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-g6m95.apps.aro.v20261001.kkqb.j8719872.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g6m95.apps.aro.v20261001.kkqb.j8719872.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:157]: failed to verify simple web app runs on v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001120bc0\u003e: \n DNS for route host agnhost-e2e-sample-app-g6m95.apps.aro.v20261001.kkqb.j8719872.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-g6m95.apps.aro.v20261001.kkqb.j8719872.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g6m95.apps.aro.v20261001.kkqb.j8719872.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-g6m95.apps.aro.v20261001.kkqb.j8719872.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-g6m95.apps.aro.v20261001.kkqb.j8719872.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g6m95.apps.aro.v20261001.kkqb.j8719872.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"6fbcff08c1d3afb706a559379ae1045751d4f0f22374c166a0e83523551a43ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102276691408719872","occurred_at":"2026-09-22T05:59:56Z","failed_at":"e2e","signature_id":"3f097e8045967cf6f7e9794619ffbd78d0b906f1fbf45cdca79e33043cc7c05c","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0016001c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-pbw5cbhdkkv4\" to finish deleting: GET https://rp.j8719872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3339f910-8502-48bd-b40c-fd8c2ee3426d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3339f910-8502-48bd-b40c-fd8c2ee3426d\",\n \"name\": \"3339f910-8502-48bd-b40c-fd8c2ee3426d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T07:11:19.031336559Z\",\n \"endTime\": \"2026-09-22T07:35:24.846401043Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svtacm4udbvukgr25lmrgtehhtlrsak still exists (deletion dispatched at 2026-09-22T07:11:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0016001c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-pbw5cbhdkkv4\" to finish deleting: GET https://rp.j8719872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3339f910-8502-48bd-b40c-fd8c2ee3426d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3339f910-8502-48bd-b40c-fd8c2ee3426d\",\n \"name\": \"3339f910-8502-48bd-b40c-fd8c2ee3426d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T07:11:19.031336559Z\",\n \"endTime\": \"2026-09-22T07:35:24.846401043Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svtacm4udbvukgr25lmrgtehhtlrsak still exists (deletion dispatched at 2026-09-22T07:11:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a2b874b4d9312d904e32f53978d511c720040c0c6845b701c5fc746c74bc1bed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102276691408719872","occurred_at":"2026-09-22T05:59:56Z","failed_at":"e2e","signature_id":"6bba268d58253a958441751f28c8663ebcc280dd195bfb45f8bdd67ce4526108","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0015800a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-fp4cw7tcx7zs\" to finish deleting: GET https://rp.j8719872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ef4b3069-c7ef-4624-b3fa-fd8d3ce178d6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ef4b3069-c7ef-4624-b3fa-fd8d3ce178d6\",\n \"name\": \"ef4b3069-c7ef-4624-b3fa-fd8d3ce178d6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T07:28:38.432300499Z\",\n \"endTime\": \"2026-09-22T07:52:38.299308463Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svtah7m0vc8lt85cerhp2ucmjjj31gs still exists (deletion dispatched at 2026-09-22T07:28:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0015800a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-fp4cw7tcx7zs\" to finish deleting: GET https://rp.j8719872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ef4b3069-c7ef-4624-b3fa-fd8d3ce178d6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ef4b3069-c7ef-4624-b3fa-fd8d3ce178d6\",\n \"name\": \"ef4b3069-c7ef-4624-b3fa-fd8d3ce178d6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T07:28:38.432300499Z\",\n \"endTime\": \"2026-09-22T07:52:38.299308463Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svtah7m0vc8lt85cerhp2ucmjjj31gs still exists (deletion dispatched at 2026-09-22T07:28:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102272452842229760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6945,"pr_state":"closed","pr_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","final_merged_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T05:43:05Z","started_at":"2026-09-22T05:43:05Z","completed_at":"2026-09-22T08:03:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T08:19:00.207947032Z","timing_metadata_checked_at":"2026-09-22T08:19:00.207947032Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T08:19:00.211008022Z","region_metadata_checked_at":"2026-09-22T08:19:00.211008022Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3878443fe87516dbbbfeb43a4f9a7391092f46ca6125f4dea2b4740377fe6aea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102272452842229760","occurred_at":"2026-09-22T05:43:05Z","failed_at":"e2e","signature_id":"8f76fd8eb82fbb3781424e84e450beb330d864e7447398473fd453d51af7bd5a","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000a3e1b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-t2xcxb7c2kws\" to finish deleting: GET https://rp.j2229760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2409cd7b-f368-4677-b635-2e440f6713ab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2409cd7b-f368-4677-b635-2e440f6713ab\",\n \"name\": \"2409cd7b-f368-4677-b635-2e440f6713ab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T07:22:36.959580513Z\",\n \"endTime\": \"2026-09-22T07:46:40.536487744Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svt7brd0t0cp9kf9q5at36rbcs9v3eb still exists (deletion dispatched at 2026-09-22T07:22:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000a3e1b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-t2xcxb7c2kws\" to finish deleting: GET https://rp.j2229760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2409cd7b-f368-4677-b635-2e440f6713ab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2409cd7b-f368-4677-b635-2e440f6713ab\",\n \"name\": \"2409cd7b-f368-4677-b635-2e440f6713ab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T07:22:36.959580513Z\",\n \"endTime\": \"2026-09-22T07:46:40.536487744Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svt7brd0t0cp9kf9q5at36rbcs9v3eb still exists (deletion dispatched at 2026-09-22T07:22:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7063/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102260929008766976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7063,"pr_state":"closed","pr_sha":"d1fadc58e79548c082fc078ecad82f622d78e654","final_merged_sha":"d1fadc58e79548c082fc078ecad82f622d78e654","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-22T04:57:18Z","started_at":"2026-09-22T04:57:18Z","completed_at":"2026-09-22T07:06:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T07:17:36.019152462Z","timing_metadata_checked_at":"2026-09-22T07:17:36.019152462Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T07:17:36.023081149Z","region_metadata_checked_at":"2026-09-22T07:17:36.023081149Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6516/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102254439539675136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6516,"pr_state":"closed","pr_sha":"571559aeb25fe3e03ab5b299d1b520ff8d3206ca","final_merged_sha":"571559aeb25fe3e03ab5b299d1b520ff8d3206ca","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T04:31:30Z","started_at":"2026-09-22T04:31:30Z","completed_at":"2026-09-22T05:10:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T05:25:47.48865423Z","timing_metadata_checked_at":"2026-09-22T05:25:47.48865423Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T05:25:47.491395921Z","region_metadata_checked_at":"2026-09-22T05:25:47.491395921Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"9d03a63e6330e705ac2c1200a142ffd0ae603f8df989fab6f00c271d9a3a3bf8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6516/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102254439539675136","occurred_at":"2026-09-22T04:31:30Z","failed_at":"provision","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"templatize-pipeline","failure_text":"time=2026-09-22T04:51:02.686Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-09-22T04:51:04.267Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-09-22T04:51:06.089Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=96622666cca941a93b9a8651dce6827ae1f27fb7972b41f9caa78871db9bc985 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9675136-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F96622666cca941a93b9a8651dce6827ae1f27fb7972b41f9caa78871db9bc985\ntime=2026-09-22T05:04:09.339Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9675136-svc/providers/Microsoft.Resources/deployments/96622666cca941a93b9a8651dce6827ae1f27fb7972b41f9caa78871db9bc985/operationStatuses/08584115542204326289\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9675136-svc/providers/Microsoft.Resources/deployments/cluster-olgmgzuso436u\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ConcurrentFederatedIdentityCredentialsWritesForSingleManagedIdentity\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Too many Federated Identity Credentials are written concurrently for the managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9675136-svc/providers/microsoft.managedidentity/userassignedidentities/fleet'. Concurrent Federated Identity Credentials writes under the same managed identity are not supported.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cb9a7ba2d967015ac741b1ba2401e95d8184541adbccfc98565936e10e7aa68c","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ConcurrentFederatedIdentityCredentialsWritesForSingleManagedIdentity; detail message Too many Federated Identity Credentials are written concurrently for the managed identity '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/microsoft.managedidentity/userassignedidentities/fleet'. Concurrent Federated Identity Credentials writes under the same managed identity are not supported.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["cb9a7ba2d967015ac741b1ba2401e95d8184541adbccfc98565936e10e7aa68c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7060/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102246390443806720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7060,"pr_state":"closed","pr_sha":"b80303ca0c8d1a96771675fae2b83405b05c8407","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T03:59:31Z","started_at":"2026-09-22T03:59:31Z","completed_at":"2026-09-22T06:22:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T06:35:59.515416855Z","timing_metadata_checked_at":"2026-09-22T06:35:59.515416855Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T06:35:59.51994164Z","region_metadata_checked_at":"2026-09-22T06:35:59.51994164Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"94c06f60af4c177b8b10d353ecefaa5f132ac8d14b18ba7299f048d1f9d2d63a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7060/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102246390443806720","occurred_at":"2026-09-22T03:59:31Z","failed_at":"e2e","signature_id":"f9be14c948f61ac7c219c13da423855cfb6ba92c1447de4905c30e0f01392252","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00168a0a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-8kr9g49k2v48\" to finish deleting: GET https://rp.j3806720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8fd82f3-9974-4eb6-997d-8bfaca303447\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8fd82f3-9974-4eb6-997d-8bfaca303447\",\n \"name\": \"c8fd82f3-9974-4eb6-997d-8bfaca303447\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T05:41:49.238086476Z\",\n \"endTime\": \"2026-09-22T06:05:52.932440268Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svrq1s2u91i6dulrlshbntsg0um43o3 still exists (deletion dispatched at 2026-09-22T05:41:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00168a0a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-8kr9g49k2v48\" to finish deleting: GET https://rp.j3806720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8fd82f3-9974-4eb6-997d-8bfaca303447\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8fd82f3-9974-4eb6-997d-8bfaca303447\",\n \"name\": \"c8fd82f3-9974-4eb6-997d-8bfaca303447\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T05:41:49.238086476Z\",\n \"endTime\": \"2026-09-22T06:05:52.932440268Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svrq1s2u91i6dulrlshbntsg0um43o3 still exists (deletion dispatched at 2026-09-22T05:41:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9689356f1c8f37195e71da3685fa05d3a333682c69e4a5f3c902b7969e0df555","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7060/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102246390443806720","occurred_at":"2026-09-22T03:59:31Z","failed_at":"e2e","signature_id":"cc4d49d2211113c6b9256ca01a1df7e2458e210a0e0b70c8dcf87367db634822","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001342060\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-8sb2nhgvgtbz\" to finish creating: GET https://rp.j3806720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0fbffb8b-7479-448a-86b5-beac660cad46\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0fbffb8b-7479-448a-86b5-beac660cad46\",\n \"name\": \"0fbffb8b-7479-448a-86b5-beac660cad46\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T05:08:13.528034946Z\",\n \"endTime\": \"2026-09-22T05:27:17.856254463Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001342060\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-8sb2nhgvgtbz\" to finish creating: GET https://rp.j3806720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0fbffb8b-7479-448a-86b5-beac660cad46\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0fbffb8b-7479-448a-86b5-beac660cad46\",\n \"name\": \"0fbffb8b-7479-448a-86b5-beac660cad46\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T05:08:13.528034946Z\",\n \"endTime\": \"2026-09-22T05:27:17.856254463Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a26bf043f64d596cec202622ba3644fb23528d332e0f8abdb980f7d2ff5cc2df","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7060/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102246390443806720","occurred_at":"2026-09-22T03:59:31Z","failed_at":"alert","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T05:14:22Z\n Ended: 2026-09-22T05:26:22Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3806720-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6891/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102236197437837312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6891,"pr_state":"closed","pr_sha":"d8bb7efc092c2e7d5ef7b195e7d9015c5f422a90","final_merged_sha":"d8bb7efc092c2e7d5ef7b195e7d9015c5f422a90","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T03:19:01Z","started_at":"2026-09-22T03:19:01Z","completed_at":"2026-09-22T05:38:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T05:58:32.322933004Z","timing_metadata_checked_at":"2026-09-22T05:58:32.322933004Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T05:58:32.326033095Z","region_metadata_checked_at":"2026-09-22T05:58:32.326033095Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"ded13369cb786bf629b7eef68e1b0b5f3135f2555a83b3510ff2162ea4f0b442","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6891/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102236197437837312","occurred_at":"2026-09-22T03:19:01Z","failed_at":"alert","signature_id":"eb32a26deada53d3c0554c1275d606596407420af9fc6a315ca6a0a9fa72f5a5","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T05:26:58Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j7837312-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-g5l6j\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-g5l6j (namespace arobit) on cluster ci01-j7837312-svc has averaged 126.4% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"e175d26b66353d8b760527cbe5fa7b86160976cd40558ed07646f76f501e6b75","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6891/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102236197437837312","occurred_at":"2026-09-22T03:19:01Z","failed_at":"e2e","signature_id":"c126cf9868b6fa1682c98b7ec463ce556108063b12012548fba4b2cbd50c1637","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0016cc0d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-hx6p7dn6kkf4\" to finish deleting: GET https://rp.j7837312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8738611c-b07f-435d-88e1-22988aa39656\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8738611c-b07f-435d-88e1-22988aa39656\",\n \"name\": \"8738611c-b07f-435d-88e1-22988aa39656\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T04:58:16.745509815Z\",\n \"endTime\": \"2026-09-22T05:22:18.875705001Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svr2l1g09bfh9s4k79s1trmkkm0nli7 still exists (deletion dispatched at 2026-09-22T04:58:18Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0016cc0d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-hx6p7dn6kkf4\" to finish deleting: GET https://rp.j7837312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8738611c-b07f-435d-88e1-22988aa39656\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8738611c-b07f-435d-88e1-22988aa39656\",\n \"name\": \"8738611c-b07f-435d-88e1-22988aa39656\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T04:58:16.745509815Z\",\n \"endTime\": \"2026-09-22T05:22:18.875705001Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svr2l1g09bfh9s4k79s1trmkkm0nli7 still exists (deletion dispatched at 2026-09-22T04:58:18Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7042/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102231676640825344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7042,"pr_state":"closed","pr_sha":"ed561d7657bd4aae970254bd806604f0b5a59888","final_merged_sha":"ed561d7657bd4aae970254bd806604f0b5a59888","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-22T03:00:48Z","started_at":"2026-09-22T03:00:48Z","completed_at":"2026-09-22T05:05:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T05:15:39.869318339Z","timing_metadata_checked_at":"2026-09-22T05:15:39.869318339Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T05:15:39.873135227Z","region_metadata_checked_at":"2026-09-22T05:15:39.873135227Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102231619598290944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7053,"pr_state":"open","pr_sha":"1c995b8c43e3ab7b347b1bdb08b62da193b79171","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T03:00:43Z","started_at":"2026-09-22T03:00:43Z","completed_at":"2026-09-22T05:19:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T05:36:44.617696626Z","timing_metadata_checked_at":"2026-09-22T05:36:44.617696626Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T05:36:44.620454118Z","region_metadata_checked_at":"2026-09-22T05:36:44.620454118Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"95cf6f74417e047c00bf8aa649584cf0216d16d3088cb584674b23fa02afc22a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102231619598290944","occurred_at":"2026-09-22T03:00:43Z","failed_at":"e2e","signature_id":"48c73f0a820a7005cd785a644d1be18a505b1fc1bc6e813c6a8d92422eec2cfd","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a24048\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-6stmg9wlrf72\" to finish deleting: GET https://rp.j8290944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/11876048-7a18-472b-ae65-ac19a08a4a96\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/11876048-7a18-472b-ae65-ac19a08a4a96\",\n \"name\": \"11876048-7a18-472b-ae65-ac19a08a4a96\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T04:37:24.350700065Z\",\n \"endTime\": \"2026-09-22T05:01:26.139604116Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svqp7o4idcshklg9rpdsdv2sclsb1mm still exists (deletion dispatched at 2026-09-22T04:37:25Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a24048\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-6stmg9wlrf72\" to finish deleting: GET https://rp.j8290944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/11876048-7a18-472b-ae65-ac19a08a4a96\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/11876048-7a18-472b-ae65-ac19a08a4a96\",\n \"name\": \"11876048-7a18-472b-ae65-ac19a08a4a96\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T04:37:24.350700065Z\",\n \"endTime\": \"2026-09-22T05:01:26.139604116Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svqp7o4idcshklg9rpdsdv2sclsb1mm still exists (deletion dispatched at 2026-09-22T04:37:25Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102231550941728768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5846,"pr_state":"closed","pr_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","final_merged_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T03:00:33Z","started_at":"2026-09-22T03:00:33Z","completed_at":"2026-09-22T05:17:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T05:36:43.866331032Z","timing_metadata_checked_at":"2026-09-22T05:36:43.866331032Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T05:36:43.86992342Z","region_metadata_checked_at":"2026-09-22T05:36:43.86992342Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"236dd4a402635ea105be7f16d2c9c8a50ca175161b19a36592f6a62bc46bf151","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102231550941728768","occurred_at":"2026-09-22T03:00:33Z","failed_at":"e2e","signature_id":"0704ec53df87d0f6d61cda9e5060a6b25d6066281db8286b2f544f0bb012d2b5","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00178c138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-dllnjkc42xws\" to finish deleting: GET https://rp.j1728768.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a2f2f18c-0a11-4b43-86e6-22cd37fa6dd4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a2f2f18c-0a11-4b43-86e6-22cd37fa6dd4\",\n \"name\": \"a2f2f18c-0a11-4b43-86e6-22cd37fa6dd4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T04:36:26.910866439Z\",\n \"endTime\": \"2026-09-22T05:00:27.577656556Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svqoe5e9dp3g5ri39rlt9i6u9dmclsp still exists (deletion dispatched at 2026-09-22T04:36:28Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00178c138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-dllnjkc42xws\" to finish deleting: GET https://rp.j1728768.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a2f2f18c-0a11-4b43-86e6-22cd37fa6dd4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a2f2f18c-0a11-4b43-86e6-22cd37fa6dd4\",\n \"name\": \"a2f2f18c-0a11-4b43-86e6-22cd37fa6dd4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T04:36:26.910866439Z\",\n \"endTime\": \"2026-09-22T05:00:27.577656556Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svqoe5e9dp3g5ri39rlt9i6u9dmclsp still exists (deletion dispatched at 2026-09-22T04:36:28Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"485b01700f5cb0fb0dfd85bba6ac90e99154c480b011e5a31d9a253494b116ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102231550941728768","occurred_at":"2026-09-22T03:00:33Z","failed_at":"alert","signature_id":"454a09f778a2169168b336e5d77f9b9de90a7344d98888a2101b0f7ede0e3c84","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T04:17:50Z\n Ended: 2026-09-22T04:50:51Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j1728768-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} exceeded 1 second over the past 30 minutes (cluster ci01-j1728768-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}},{"row_id":"c954a601aefe0c0aa32917663cc26ef59ac5e9052fc8ac04655553c45b1102a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102231550941728768","occurred_at":"2026-09-22T03:00:33Z","failed_at":"alert","signature_id":"7b40679fed1158a25c62a0c7e6a9546444082fb4c154448640ebd2d79aa45c65","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T04:51:26Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j1728768-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-wdz85\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-wdz85 (namespace arobit) on cluster ci01-j1728768-svc has averaged 125.8% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102231517219524608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6945,"pr_state":"closed","pr_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","final_merged_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T03:00:25Z","started_at":"2026-09-22T03:00:25Z","completed_at":"2026-09-22T05:25:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T05:36:44.074373966Z","timing_metadata_checked_at":"2026-09-22T05:36:44.074373966Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T05:36:44.07943105Z","region_metadata_checked_at":"2026-09-22T05:36:44.07943105Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"521fa2215fb63792008fcbf6bdad0bbb20366650fcca0fc092c5ea360a2c6e8a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102231517219524608","occurred_at":"2026-09-22T03:00:25Z","failed_at":"e2e","signature_id":"3ac4aba76510c25db23049b52ca845fdda949e4eb97a61d8fe42b96c02253216","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001624078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-65cd4v\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-p6445jwrw8px\" to finish deleting: GET https://rp.j9524608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a8da0d4c-832a-4ebe-83fe-0ef1ea20c2ed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a8da0d4c-832a-4ebe-83fe-0ef1ea20c2ed\",\n \"name\": \"a8da0d4c-832a-4ebe-83fe-0ef1ea20c2ed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T04:21:02.250571798Z\",\n \"endTime\": \"2026-09-22T04:45:04.991809002Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svqpgqbhn074kn1grepral244a0uo9a still exists (deletion dispatched at 2026-09-22T04:21:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001624078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-65cd4v\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-p6445jwrw8px\" to finish deleting: GET https://rp.j9524608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a8da0d4c-832a-4ebe-83fe-0ef1ea20c2ed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a8da0d4c-832a-4ebe-83fe-0ef1ea20c2ed\",\n \"name\": \"a8da0d4c-832a-4ebe-83fe-0ef1ea20c2ed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T04:21:02.250571798Z\",\n \"endTime\": \"2026-09-22T04:45:04.991809002Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svqpgqbhn074kn1grepral244a0uo9a still exists (deletion dispatched at 2026-09-22T04:21:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5b8fcd303a0c03c39d30350a7b7c800710bdf0c4144b6de49afe6f991d1850b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102231517219524608","occurred_at":"2026-09-22T03:00:25Z","failed_at":"e2e","signature_id":"71286f605bf6e11a92c15b43ed0ece7a96a19a4336765536c2ae052a35632578","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0015ee120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-dvbs4m\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-8dfdxdjq6sl7\" to finish deleting: GET https://rp.j9524608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dfeb1299-93f0-4935-b7a6-3aa4e0368f83\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dfeb1299-93f0-4935-b7a6-3aa4e0368f83\",\n \"name\": \"dfeb1299-93f0-4935-b7a6-3aa4e0368f83\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T04:21:06.047683642Z\",\n \"endTime\": \"2026-09-22T04:45:14.955489839Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svqpgvvjijkhsd79om9jjphnksski1h still exists (deletion dispatched at 2026-09-22T04:21:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0015ee120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-dvbs4m\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-8dfdxdjq6sl7\" to finish deleting: GET https://rp.j9524608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dfeb1299-93f0-4935-b7a6-3aa4e0368f83\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dfeb1299-93f0-4935-b7a6-3aa4e0368f83\",\n \"name\": \"dfeb1299-93f0-4935-b7a6-3aa4e0368f83\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T04:21:06.047683642Z\",\n \"endTime\": \"2026-09-22T04:45:14.955489839Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svqpgvvjijkhsd79om9jjphnksski1h still exists (deletion dispatched at 2026-09-22T04:21:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"77b0032050a22250aee6956e88d7e48117d4f82a006b5eead99c8bc2abb1b46f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102231517219524608","occurred_at":"2026-09-22T03:00:25Z","failed_at":"e2e","signature_id":"2e08a12fbaab745f1c9ad8675e56cc4e1764db69fce02141a843f86421385411","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001074030\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-4pj899qpn6jc\" to finish deleting: GET https://rp.j9524608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7b12757b-9d14-428d-b673-87c01137c0b4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7b12757b-9d14-428d-b673-87c01137c0b4\",\n \"name\": \"7b12757b-9d14-428d-b673-87c01137c0b4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T04:32:27.654860692Z\",\n \"endTime\": \"2026-09-22T04:56:34.502862674Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svqpiu74flsdu8i1gba51b7sq6qn8pc still exists (deletion dispatched at 2026-09-22T04:32:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001074030\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-4pj899qpn6jc\" to finish deleting: GET https://rp.j9524608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7b12757b-9d14-428d-b673-87c01137c0b4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7b12757b-9d14-428d-b673-87c01137c0b4\",\n \"name\": \"7b12757b-9d14-428d-b673-87c01137c0b4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T04:32:27.654860692Z\",\n \"endTime\": \"2026-09-22T04:56:34.502862674Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svqpiu74flsdu8i1gba51b7sq6qn8pc still exists (deletion dispatched at 2026-09-22T04:32:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e4edf4faf2cd16e831435944cad6207896a51d5724c3b573ce4326f1048da819","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102231517219524608","occurred_at":"2026-09-22T03:00:25Z","failed_at":"e2e","signature_id":"ad7859705e4441294e9606a8e1e2039a0669249b9198f60c803017117efd2c42","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a62078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-g5q4pnrvzzcw\" to finish deleting: GET https://rp.j9524608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cebb22bc-25cc-4fe0-9947-7179f9b8772b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cebb22bc-25cc-4fe0-9947-7179f9b8772b\",\n \"name\": \"cebb22bc-25cc-4fe0-9947-7179f9b8772b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T04:43:42.181198275Z\",\n \"endTime\": \"2026-09-22T05:07:44.535589725Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svqpft6nt92r6veu7aabqm40942hic4 still exists (deletion dispatched at 2026-09-22T04:43:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a62078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-g5q4pnrvzzcw\" to finish deleting: GET https://rp.j9524608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cebb22bc-25cc-4fe0-9947-7179f9b8772b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cebb22bc-25cc-4fe0-9947-7179f9b8772b\",\n \"name\": \"cebb22bc-25cc-4fe0-9947-7179f9b8772b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T04:43:42.181198275Z\",\n \"endTime\": \"2026-09-22T05:07:44.535589725Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svqpft6nt92r6veu7aabqm40942hic4 still exists (deletion dispatched at 2026-09-22T04:43:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102230542270337024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5724,"pr_state":"closed","pr_sha":"016fdecacb5c2b53cdf4e1d7f9db9356d994bc7c","final_merged_sha":"02f55855ad2e600a742ce1b551002c7fbd872350","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T02:56:33Z","started_at":"2026-09-22T02:56:33Z","completed_at":"2026-09-22T05:09:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T05:25:47.466256Z","timing_metadata_checked_at":"2026-09-22T05:25:47.466256Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T05:25:47.469847989Z","region_metadata_checked_at":"2026-09-22T05:25:47.469847989Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"dc74aade4618e49f7beee06fa58293e187cb82318929203bf171bb2e3f03582b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102230542270337024","occurred_at":"2026-09-22T02:56:33Z","failed_at":"e2e","signature_id":"8659fb6913fb2dfef84644160b413f5872d6566f70a0cee56ca0e1618ca42d3e","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001262150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-mm8687thvtjz\" to finish deleting: GET https://rp.j0337024.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b787e318-ad3e-4acc-bfcf-e4f4c39ce55a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b787e318-ad3e-4acc-bfcf-e4f4c39ce55a\",\n \"name\": \"b787e318-ad3e-4acc-bfcf-e4f4c39ce55a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T04:30:32.899691516Z\",\n \"endTime\": \"2026-09-22T04:54:37.595087082Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svqmjoqgfjcv3n68g00396ud2hcevk5 still exists (deletion dispatched at 2026-09-22T04:30:34Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001262150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-mm8687thvtjz\" to finish deleting: GET https://rp.j0337024.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b787e318-ad3e-4acc-bfcf-e4f4c39ce55a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b787e318-ad3e-4acc-bfcf-e4f4c39ce55a\",\n \"name\": \"b787e318-ad3e-4acc-bfcf-e4f4c39ce55a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T04:30:32.899691516Z\",\n \"endTime\": \"2026-09-22T04:54:37.595087082Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svqmjoqgfjcv3n68g00396ud2hcevk5 still exists (deletion dispatched at 2026-09-22T04:30:34Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102230390138736640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T02:55:57Z","started_at":"2026-09-22T02:55:57Z","completed_at":"2026-09-22T05:22:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T05:36:44.494139222Z","timing_metadata_checked_at":"2026-09-22T05:36:44.494139222Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T05:36:44.497147012Z","region_metadata_checked_at":"2026-09-22T05:36:44.497147012Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"3759f0be89ef35baea2261c84c56afb99e2a29d6a4ed943826fc6b4874d734f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102230390138736640","occurred_at":"2026-09-22T02:55:57Z","failed_at":"alert","signature_id":"e4f7ed3558fb4fd8a7cd5814747918a0c71dabdb7eba09fe811038658bae7ec6","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T03:59:08Z\n Ended: 2026-09-22T04:33:08Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j8736640-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j8736640-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"45ce3e091bd783701b12e1bb90dfd6f71076549abc9ad438090329a355a0efc0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102230390138736640","occurred_at":"2026-09-22T02:55:57Z","failed_at":"e2e","signature_id":"75046b60e1f2f39e795e5adb5230e5ac2986a3bec4514e7d4669a5e735bbaada","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012dc0a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-sw29fbqg4jf9\" to finish deleting: GET https://rp.j8736640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a5c5a2a-f5fb-43d1-be90-346fb5dab267\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a5c5a2a-f5fb-43d1-be90-346fb5dab267\",\n \"name\": \"5a5c5a2a-f5fb-43d1-be90-346fb5dab267\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T04:41:00.185282796Z\",\n \"endTime\": \"2026-09-22T05:05:08.702721718Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svqqrb0hjk2rjgf2447lvn1fit1l09o still exists (deletion dispatched at 2026-09-22T04:41:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012dc0a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-sw29fbqg4jf9\" to finish deleting: GET https://rp.j8736640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a5c5a2a-f5fb-43d1-be90-346fb5dab267\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a5c5a2a-f5fb-43d1-be90-346fb5dab267\",\n \"name\": \"5a5c5a2a-f5fb-43d1-be90-346fb5dab267\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T04:41:00.185282796Z\",\n \"endTime\": \"2026-09-22T05:05:08.702721718Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svqqrb0hjk2rjgf2447lvn1fit1l09o still exists (deletion dispatched at 2026-09-22T04:41:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7062/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102226956232691712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7062,"pr_state":"closed","pr_sha":"c608bc0f67938a84348cd9f8fd914b66762aa966","final_merged_sha":"c608bc0f67938a84348cd9f8fd914b66762aa966","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T02:42:18Z","started_at":"2026-09-22T02:42:18Z","completed_at":"2026-09-22T03:25:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T03:40:46.500690039Z","timing_metadata_checked_at":"2026-09-22T03:40:46.500690039Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T03:40:46.504307327Z","region_metadata_checked_at":"2026-09-22T03:40:46.504307327Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"fc0ce9ed14610dd45de403d13e68fabb4936342d6d66b9c975eec8cd3fdac91b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7062/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102226956232691712","occurred_at":"2026-09-22T02:42:18Z","failed_at":"provision","signature_id":"d0835b4a87efe485e5d2632604de682cfdfca8dd7d942ad8c07295231be37bc7","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/admin-api-cert","test_suite":"templatize-pipeline","failure_text":"time=2026-09-22T03:15:16.448Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert description=\"Step admin-api-cert\\n Kind: CreateCertificate\\n\"\ntime=2026-09-22T03:15:16.448Z level=INFO msg=\"Creating certificate\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert vaultBaseUrl=https://aro-hcp-dev-svc-kv.vault.azure.net/ certificateName=admin-api-cert-ci01-j2691712 issuer=Self commonName=admin.j2691712.hcpsvc.osadev.cloud san=admin.j2691712.hcpsvc.osadev.cloud\ntime=2026-09-22T03:15:17.222Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert err=\"error running Create Certificate Step: failed to create certificate \\\"admin-api-cert-ci01-j2691712\\\" in vault \\\"https://aro-hcp-dev-svc-kv.vault.azure.net/\\\": POST https://aro-hcp-dev-svc-kv.vault.azure.net/certificates/admin-api-cert-ci01-j2691712/create\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Certificate admin-api-cert-ci01-j2691712 is currently in a deleted but recoverable state, and its name cannot be reused; in this state, the certificate can only be recovered or purged.\\\",\\n \\\"innererror\\\": {\\n \\\"code\\\": \\\"ObjectIsDeletedButRecoverable\\\"\\n }\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6f6e7e66ad8bc42bd6647f1e246b1e77205441e0c9495974d9b1f940d8bdec9c","failure_pattern":"ERROR CODE: Conflict; detail code ObjectIsDeletedButRecoverable","search_query":"ERROR CODE: Conflict"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6f6e7e66ad8bc42bd6647f1e246b1e77205441e0c9495974d9b1f940d8bdec9c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7060/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102210924612947968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7060,"pr_state":"closed","pr_sha":"b80303ca0c8d1a96771675fae2b83405b05c8407","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T01:38:36Z","started_at":"2026-09-22T01:38:36Z","completed_at":"2026-09-22T03:46:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T04:01:46.442692094Z","timing_metadata_checked_at":"2026-09-22T04:01:46.442692094Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T04:01:46.446042383Z","region_metadata_checked_at":"2026-09-22T04:01:46.446042383Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"6a6bb69f9e4749d270dd8803ce991a59df059e7f6db098395d29edffe9a0e3b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7060/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102210924612947968","occurred_at":"2026-09-22T01:38:36Z","failed_at":"e2e","signature_id":"12855f5c94d8ce8642a8edba1fdb0e1255692259e41e52b94faff68410c3c55b","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a441e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-gkxv2rdhffbn\" to finish deleting: GET https://rp.j2947968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/539c9b60-22fe-4178-8a04-19adb0069261\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/539c9b60-22fe-4178-8a04-19adb0069261\",\n \"name\": \"539c9b60-22fe-4178-8a04-19adb0069261\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T03:04:45.070729658Z\",\n \"endTime\": \"2026-09-22T03:28:49.639377463Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svpfbrcndrja8lh6o4gbf38h9p1ph6m still exists (deletion dispatched at 2026-09-22T03:04:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a441e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-gkxv2rdhffbn\" to finish deleting: GET https://rp.j2947968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/539c9b60-22fe-4178-8a04-19adb0069261\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/539c9b60-22fe-4178-8a04-19adb0069261\",\n \"name\": \"539c9b60-22fe-4178-8a04-19adb0069261\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T03:04:45.070729658Z\",\n \"endTime\": \"2026-09-22T03:28:49.639377463Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svpfbrcndrja8lh6o4gbf38h9p1ph6m still exists (deletion dispatched at 2026-09-22T03:04:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6d682d88c699904c54f3f9fe8571de50fc33112f40e060ab431dec65b9320776","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7060/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102210924612947968","occurred_at":"2026-09-22T01:38:36Z","failed_at":"e2e","signature_id":"c8393314a20132a3e98fc6cd81c0dae2452cf2d8e1b2e7e97ae3de2436c44317","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0015a65b8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-sjts2p\" in resourcegroup=\"rg-np-skip-minor-sjts2p-xgpmp2tpf5bv\" to finish deleting: GET https://rp.j2947968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d26277a6-0af9-4da0-80f5-448652226fff\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d26277a6-0af9-4da0-80f5-448652226fff\",\n \"name\": \"d26277a6-0af9-4da0-80f5-448652226fff\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T03:01:26.657883091Z\",\n \"endTime\": \"2026-09-22T03:25:29.637165858Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svpfbucgbjrpto0mrulh6vn6q4os1c1 still exists (deletion dispatched at 2026-09-22T03:01:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0015a65b8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-sjts2p\" in resourcegroup=\"rg-np-skip-minor-sjts2p-xgpmp2tpf5bv\" to finish deleting: GET https://rp.j2947968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d26277a6-0af9-4da0-80f5-448652226fff\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d26277a6-0af9-4da0-80f5-448652226fff\",\n \"name\": \"d26277a6-0af9-4da0-80f5-448652226fff\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T03:01:26.657883091Z\",\n \"endTime\": \"2026-09-22T03:25:29.637165858Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svpfbucgbjrpto0mrulh6vn6q4os1c1 still exists (deletion dispatched at 2026-09-22T03:01:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a2ce863c31026c6bf14546377ab847d5b43f68d66a311c315f0c4e1f8f2bf850","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7060/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102210924612947968","occurred_at":"2026-09-22T01:38:36Z","failed_at":"alert","signature_id":"f890e54a3771983140c6e7e468f8358fcb9025cd81792884f77a2b7344615b44","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T03:24:14Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j2947968-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-49lgs\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-49lgs (namespace arobit) on cluster ci01-j2947968-svc has averaged 126.8% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"b8bccee9303e80509e1697fd744a2b23d9e6e900305de00be95e7d273e4547e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7060/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102210924612947968","occurred_at":"2026-09-22T01:38:36Z","failed_at":"e2e","signature_id":"a8ff04f71f30ac8836925032560c4015403e6d160d584e18c3135ac999e36302","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000b302a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-rmtxh2ts9kfb\" to finish deleting: GET https://rp.j2947968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/586ad5d7-78c5-445c-8754-3de3e7992633\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/586ad5d7-78c5-445c-8754-3de3e7992633\",\n \"name\": \"586ad5d7-78c5-445c-8754-3de3e7992633\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T02:50:05.627831093Z\",\n \"endTime\": \"2026-09-22T03:14:06.922141251Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svpfn4442k3esfkats1kkvuno9gqgd0 still exists (deletion dispatched at 2026-09-22T02:50:07Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000b302a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-rmtxh2ts9kfb\" to finish deleting: GET https://rp.j2947968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/586ad5d7-78c5-445c-8754-3de3e7992633\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/586ad5d7-78c5-445c-8754-3de3e7992633\",\n \"name\": \"586ad5d7-78c5-445c-8754-3de3e7992633\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T02:50:05.627831093Z\",\n \"endTime\": \"2026-09-22T03:14:06.922141251Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svpfn4442k3esfkats1kkvuno9gqgd0 still exists (deletion dispatched at 2026-09-22T02:50:07Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7061/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102203845328244736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7061,"pr_state":"closed","pr_sha":"52b8f229469c6935a54072331a3683c04e2e145a","final_merged_sha":"52b8f229469c6935a54072331a3683c04e2e145a","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T01:10:26Z","started_at":"2026-09-22T01:10:26Z","completed_at":"2026-09-22T03:12:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T03:19:54.642565643Z","timing_metadata_checked_at":"2026-09-22T03:19:54.642565643Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T03:19:54.645504234Z","region_metadata_checked_at":"2026-09-22T03:19:54.645504234Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"69b7072e6310e0465b9cf1832f0f92ba1a18d2f25047d1a3ed20e1cc3aa3400e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7061/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102203845328244736","occurred_at":"2026-09-22T01:10:26Z","failed_at":"e2e","signature_id":"12732a5ca59d07fe7708e2bc28f4d0a32b1968e01ba973942f97caff43c12fc7","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0013ac0c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-bx6lqvftk4w7\" to finish deleting: GET https://rp.j8244736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/151984c8-dc03-45fe-a98d-c1ec95b80b1b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/151984c8-dc03-45fe-a98d-c1ec95b80b1b\",\n \"name\": \"151984c8-dc03-45fe-a98d-c1ec95b80b1b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T02:31:50.927530568Z\",\n \"endTime\": \"2026-09-22T02:55:59.602648065Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svp6bj50tirtgu0edlj3fjaggm8e3pr still exists (deletion dispatched at 2026-09-22T02:31:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0013ac0c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-bx6lqvftk4w7\" to finish deleting: GET https://rp.j8244736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/151984c8-dc03-45fe-a98d-c1ec95b80b1b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/151984c8-dc03-45fe-a98d-c1ec95b80b1b\",\n \"name\": \"151984c8-dc03-45fe-a98d-c1ec95b80b1b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T02:31:50.927530568Z\",\n \"endTime\": \"2026-09-22T02:55:59.602648065Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svp6bj50tirtgu0edlj3fjaggm8e3pr still exists (deletion dispatched at 2026-09-22T02:31:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7059/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102191495971868672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7059,"pr_state":"closed","pr_sha":"7d175e1ddb25678fa96328e62b2528d9aca28576","final_merged_sha":"b22cddf8453a7f89c54cd40aafd6d1bcecefc802","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T00:21:24Z","started_at":"2026-09-22T00:21:24Z","completed_at":"2026-09-22T02:34:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T02:38:26.499548555Z","timing_metadata_checked_at":"2026-09-22T02:38:26.499548555Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T02:38:26.504748239Z","region_metadata_checked_at":"2026-09-22T02:38:26.504748239Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"4fc6426ac07e138b40edee5d62116dc341617b97bf31e0a4c8bf84e4aa92a601","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7059/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102191495971868672","occurred_at":"2026-09-22T00:21:24Z","failed_at":"alert","signature_id":"23b506fc45f1e15d2a4b76c5499af25f7494f8b605bccf9085abe7a28964c72e","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T02:22:33Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j1868672-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-x2z5m\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-x2z5m (namespace arobit) on cluster ci01-j1868672-svc has averaged 126.3% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"9104f6885995992a65b595338fd74de66f41824425ab86f92aec18a518cc988b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7059/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102191495971868672","occurred_at":"2026-09-22T00:21:24Z","failed_at":"e2e","signature_id":"4d055ed76535ebb1f15e7aca5a47519d6e1a265cb5ef2d99a239a68927a314ab","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011d4108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-rlrmtm9g69c6\" to finish deleting: GET https://rp.j1868672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b5f8d503-82d0-44c5-b405-81069f91db8c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b5f8d503-82d0-44c5-b405-81069f91db8c\",\n \"name\": \"b5f8d503-82d0-44c5-b405-81069f91db8c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T01:52:36.957024286Z\",\n \"endTime\": \"2026-09-22T02:16:41.168615942Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svocvbp6291hdqpj2pim2m0gvn0ml5d still exists (deletion dispatched at 2026-09-22T01:52:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011d4108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-rlrmtm9g69c6\" to finish deleting: GET https://rp.j1868672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b5f8d503-82d0-44c5-b405-81069f91db8c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b5f8d503-82d0-44c5-b405-81069f91db8c\",\n \"name\": \"b5f8d503-82d0-44c5-b405-81069f91db8c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T01:52:36.957024286Z\",\n \"endTime\": \"2026-09-22T02:16:41.168615942Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svocvbp6291hdqpj2pim2m0gvn0ml5d still exists (deletion dispatched at 2026-09-22T01:52:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7042/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102186818521796608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7042,"pr_state":"closed","pr_sha":"ed561d7657bd4aae970254bd806604f0b5a59888","final_merged_sha":"ed561d7657bd4aae970254bd806604f0b5a59888","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T00:02:48Z","started_at":"2026-09-22T00:02:48Z","completed_at":"2026-09-22T00:39:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T00:43:46.893747869Z","timing_metadata_checked_at":"2026-09-22T00:43:46.893747869Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T00:43:46.897311858Z","region_metadata_checked_at":"2026-09-22T00:43:46.897311858Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c3c8753796dcde37abef6120285ae53a28080cb098d91035d0591f6110273270","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7042/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102186818521796608","occurred_at":"2026-09-22T00:02:48Z","failed_at":"provision","signature_id":"95e7be5f64e9ad50a9e7306687aca3408d8573d35d0eb9459a8bc2935088f19f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/admin-api-cert","test_suite":"templatize-pipeline","failure_text":"time=2026-09-22T00:26:51.703Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert description=\"Step admin-api-cert\\n Kind: CreateCertificate\\n\"\ntime=2026-09-22T00:26:51.703Z level=INFO msg=\"Creating certificate\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert vaultBaseUrl=https://aro-hcp-dev-svc-kv.vault.azure.net/ certificateName=admin-api-cert-ci01-j1796608 issuer=Self commonName=admin.j1796608.hcpsvc.osadev.cloud san=admin.j1796608.hcpsvc.osadev.cloud\ntime=2026-09-22T00:26:52.483Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert err=\"error running Create Certificate Step: failed to create certificate \\\"admin-api-cert-ci01-j1796608\\\" in vault \\\"https://aro-hcp-dev-svc-kv.vault.azure.net/\\\": POST https://aro-hcp-dev-svc-kv.vault.azure.net/certificates/admin-api-cert-ci01-j1796608/create\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Certificate admin-api-cert-ci01-j1796608 is currently in a deleted but recoverable state, and its name cannot be reused; in this state, the certificate can only be recovered or purged.\\\",\\n \\\"innererror\\\": {\\n \\\"code\\\": \\\"ObjectIsDeletedButRecoverable\\\"\\n }\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6f6e7e66ad8bc42bd6647f1e246b1e77205441e0c9495974d9b1f940d8bdec9c","failure_pattern":"ERROR CODE: Conflict; detail code ObjectIsDeletedButRecoverable","search_query":"ERROR CODE: Conflict"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6f6e7e66ad8bc42bd6647f1e246b1e77205441e0c9495974d9b1f940d8bdec9c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102186809273356288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7053,"pr_state":"open","pr_sha":"1c995b8c43e3ab7b347b1bdb08b62da193b79171","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-22T00:02:45Z","started_at":"2026-09-22T00:02:45Z","completed_at":"2026-09-22T02:13:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T02:18:06.652337267Z","timing_metadata_checked_at":"2026-09-22T02:18:06.652337267Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T02:18:06.655308158Z","region_metadata_checked_at":"2026-09-22T02:18:06.655308158Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"08a8eb9157d95ccaa7b6a8adbbb2483e8aefd45592d9fac901420216abe78655","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102186809273356288","occurred_at":"2026-09-22T00:02:45Z","failed_at":"e2e","signature_id":"c801a40928cf7ce27dc4dc008fa87ca8f70bc430ef051fbe1974207aae546b74","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012c20d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-dg-minor-frdg8x\" in resourcegroup=\"rg-np-dg-minor-frdg8x-bmvs69lrn9nj\" to finish deleting: GET https://rp.j3356288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e0f3f148-543d-4e5c-9517-ece99016d1a5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e0f3f148-543d-4e5c-9517-ece99016d1a5\",\n \"name\": \"e0f3f148-543d-4e5c-9517-ece99016d1a5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T01:33:09.939984258Z\",\n \"endTime\": \"2026-09-22T01:57:11.486464494Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svo54ortklo3gj9q24qi42opdhtistm still exists (deletion dispatched at 2026-09-22T01:33:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012c20d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-dg-minor-frdg8x\" in resourcegroup=\"rg-np-dg-minor-frdg8x-bmvs69lrn9nj\" to finish deleting: GET https://rp.j3356288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e0f3f148-543d-4e5c-9517-ece99016d1a5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e0f3f148-543d-4e5c-9517-ece99016d1a5\",\n \"name\": \"e0f3f148-543d-4e5c-9517-ece99016d1a5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T01:33:09.939984258Z\",\n \"endTime\": \"2026-09-22T01:57:11.486464494Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svo54ortklo3gj9q24qi42opdhtistm still exists (deletion dispatched at 2026-09-22T01:33:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"683f2eab204b3d4d347490cb6852f9f4ff5e3576994eb1763ba417a1b271b610","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102186809273356288","occurred_at":"2026-09-22T00:02:45Z","failed_at":"alert","signature_id":"8c975fd9f886200ee26bc3444b0583309d21a1e572b61a74aaf57cf9297a81e7","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-22T01:34:08Z\n Ended: 2026-09-22T01:43:06Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j3356288-svc\", component=\"frontend\", method=\"delete\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: More than 5% of frontend requests for delete /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j3356288-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102186737458483200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6945,"pr_state":"closed","pr_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","final_merged_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T00:02:29Z","started_at":"2026-09-22T00:02:29Z","completed_at":"2026-09-22T02:17:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T02:28:11.143154804Z","timing_metadata_checked_at":"2026-09-22T02:28:11.143154804Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T02:28:11.159584052Z","region_metadata_checked_at":"2026-09-22T02:28:11.159584052Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"29861ad019c5a106127601e45745e47996e778a4635b8bc1279c6519b6f993bb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102186737458483200","occurred_at":"2026-09-22T00:02:29Z","failed_at":"e2e","signature_id":"60822c0ff2befe11e2e3b0e3ca8a5d85423c4687184b8ad104d0ffbc18cd2b02","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001404e70\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-xjshj2\" in resourcegroup=\"rg-np-version-upgrade-xjshj2-h4xgk6nl5j9d\" to finish deleting: GET https://rp.j8483200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1a98cfab-0566-40a0-9178-58b4073d509a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1a98cfab-0566-40a0-9178-58b4073d509a\",\n \"name\": \"1a98cfab-0566-40a0-9178-58b4073d509a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T01:33:50.788314397Z\",\n \"endTime\": \"2026-09-22T01:57:55.348577083Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svo4230k0qs66paonqn2e2dbtlcusvd still exists (deletion dispatched at 2026-09-22T01:33:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001404e70\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-xjshj2\" in resourcegroup=\"rg-np-version-upgrade-xjshj2-h4xgk6nl5j9d\" to finish deleting: GET https://rp.j8483200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1a98cfab-0566-40a0-9178-58b4073d509a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1a98cfab-0566-40a0-9178-58b4073d509a\",\n \"name\": \"1a98cfab-0566-40a0-9178-58b4073d509a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T01:33:50.788314397Z\",\n \"endTime\": \"2026-09-22T01:57:55.348577083Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svo4230k0qs66paonqn2e2dbtlcusvd still exists (deletion dispatched at 2026-09-22T01:33:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"310108aa5920279122e29a1d7b63273ebc5e706841a554935ec5cbc945a3edc1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102186737458483200","occurred_at":"2026-09-22T00:02:29Z","failed_at":"alert","signature_id":"0d5065ea065f8b5d5b038c244627dc339027669eafe53ebeb82258c8b157738e","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-22T01:58:50Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j8483200-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-pgmn8\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-pgmn8 (namespace arobit) on cluster ci01-j8483200-svc has averaged 126.2% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"dd152dcfb6557be1820186e0636fa5d217badb1c2dc99b911d386a687099c8ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102186737458483200","occurred_at":"2026-09-22T00:02:29Z","failed_at":"e2e","signature_id":"e1f75fdde1658fe53d9052beb10bdbedb5f8af112cd433dcfd1ac576c7fd2884","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001880030\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-w8lc4ztvgcdz\" to finish deleting: GET https://rp.j8483200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ab28a42-8341-418d-96d7-128339122a83\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ab28a42-8341-418d-96d7-128339122a83\",\n \"name\": \"9ab28a42-8341-418d-96d7-128339122a83\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T01:36:01.895564169Z\",\n \"endTime\": \"2026-09-22T02:00:05.356533198Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svo404qsimsek2clk2adqccvekdokcj still exists (deletion dispatched at 2026-09-22T01:36:03Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001880030\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-w8lc4ztvgcdz\" to finish deleting: GET https://rp.j8483200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ab28a42-8341-418d-96d7-128339122a83\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ab28a42-8341-418d-96d7-128339122a83\",\n \"name\": \"9ab28a42-8341-418d-96d7-128339122a83\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T01:36:01.895564169Z\",\n \"endTime\": \"2026-09-22T02:00:05.356533198Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2svo404qsimsek2clk2adqccvekdokcj still exists (deletion dispatched at 2026-09-22T01:36:03Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102186730835677184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5846,"pr_state":"closed","pr_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","final_merged_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T00:02:27Z","started_at":"2026-09-22T00:02:27Z","completed_at":"2026-09-22T00:43:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T01:04:18.404385476Z","timing_metadata_checked_at":"2026-09-22T01:04:18.404385476Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T01:04:18.407057068Z","region_metadata_checked_at":"2026-09-22T01:04:18.407057068Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"a1e195f4a5e631cdb7e83943064044f79bdc074d594a455a56c6470dde89cab0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102186730835677184","occurred_at":"2026-09-22T00:02:27Z","failed_at":"provision","signature_id":"a25c2fb7b5fe7b3b703abe9fe9cdab41f229d7afd7a57f710d0cdf443b52a3a0","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-22T00:28:42.028Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-09-22T00:31:19.046Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-22T00:28:42.035099926Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5677184-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5677184-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-09-22T00:28:44.199184892Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5677184-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5677184-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-22T00:28:44.294775263Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5677184-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5677184-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T00:31:19.041004649Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5677184-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5677184-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"B5981CAA76057630\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5677184-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5677184-mgmt-1\\\",\\\"resourceName\\\":\\\"ci01-j5677184-mgmt-1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-22T00:28:49.2033491Z\\\",\\\"duration\\\":\\\"PT2M28.4887595S\\\",\\\"statusCode\\\":\\\"Conflict\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ReconcileVMSSAgentPoolFailed\\\",\\\"message\\\":\\\"Internal server error\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-22T00:31:19.041107186Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5677184-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5677184-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5677184-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584115699609930281\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5677184-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5677184-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5677184-mgmt-1\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ReconcileVMSSAgentPoolFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Internal server error\\\\\\\"\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"2m37.00601594s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"974128b582e7e400d6215f2a944dca5aafec67820d9ccf78df791931926850f4","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ReconcileVMSSAgentPoolFailed; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["974128b582e7e400d6215f2a944dca5aafec67820d9ccf78df791931926850f4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]},{"environment":"int","summary":{"total_runs":3,"failed_runs":1,"runs_with_occurrences":1,"runs_with_failure_pattern_matches":1,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":2,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2102492733552726016","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T19:50:05Z","started_at":"2026-09-22T19:50:05Z","completed_at":"2026-09-22T22:54:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T23:02:37.277097945Z","timing_metadata_checked_at":"2026-09-22T23:02:37.277097945Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T23:02:37.280193334Z","region_metadata_checked_at":"2026-09-22T23:02:37.280193334Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"13021e331e78193d45f3b5e303b7b770f579586bb46d0b1b3ab8e7033ae31a0f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2102492733552726016","occurred_at":"2026-09-22T19:50:05Z","failed_at":"e2e","signature_id":"6299df96b3ee9add016c9b5bb17a4bd6b607c208223caa2e182fd6c3ce6bd576","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-h7c9rnh9n24x/cluster-candidate-4-21-jthhdc should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00102a600\u003e: \n failed to create HCP cluster cluster-candidate-4-21-jthhdc: failed waiting for cluster=\"cluster-candidate-4-21-jthhdc\" in resourcegroup=\"rg-candidate-4-21-h7c9rnh9n24x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b55cb22c-9a55-40c5-b3c6-893ec03a9fc6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b55cb22c-9a55-40c5-b3c6-893ec03a9fc6\",\n \"name\": \"b55cb22c-9a55-40c5-b3c6-893ec03a9fc6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T21:07:20.636049246Z\",\n \"endTime\": \"2026-09-22T21:36:27.958431734Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 2m58s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-h7c9rnh9n24x/cluster-candidate-4-21-jthhdc should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00102a600\u003e: \n failed to create HCP cluster cluster-candidate-4-21-jthhdc: failed waiting for cluster=\"cluster-candidate-4-21-jthhdc\" in resourcegroup=\"rg-candidate-4-21-h7c9rnh9n24x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b55cb22c-9a55-40c5-b3c6-893ec03a9fc6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b55cb22c-9a55-40c5-b3c6-893ec03a9fc6\",\n \"name\": \"b55cb22c-9a55-40c5-b3c6-893ec03a9fc6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T21:07:20.636049246Z\",\n \"endTime\": \"2026-09-22T21:36:27.958431734Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 2m58s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"38eeaeccffa0491180dfc6054cc1b3abf572643c67dff782fd1fdc281074fb46","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3a76866c9b0f564f3076936726d48d0fa30d851b881be95820ca0b9d6210053b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2102492733552726016","occurred_at":"2026-09-22T19:50:05Z","failed_at":"e2e","signature_id":"6436a970a48f30cfe1ee0c5e3a5720711b6021d2b00089bf5fd8c321a3203f96","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:159]: failed to install Cilium chart via Helm\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008581e0\u003e: \n failed to locate cilium chart: Get \"https://helm.cilium.io/cilium-1.19.2.tgz\": dial tcp [2606:50c0:8000::153]:443: connect: network is unreachable\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:159]: failed to install Cilium chart via Helm\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008581e0\u003e: \n failed to locate cilium chart: Get \"https://helm.cilium.io/cilium-1.19.2.tgz\": dial tcp [2606:50c0:8000::153]:443: connect: network is unreachable\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"351f45356c048233596e7dd87add23f59e5ae9c144a3196c0577133fac0ad667","failure_pattern":"failed to locate cilium chart: Get \"\u003curl\u003e\": dial tcp \u003cip\u003e: connect: network is unreachable","search_query":"failed to locate cilium chart: Get \"https://helm.cilium.io/cilium-1.19.2.tgz\": dial tcp [2606:50c0:8000::153]:443: connect: network is unreachable"}},{"row_id":"67a1ce539f7c04ead495324ddd77ceb9c7e98df01ed77cd856a65ba8d51e2915","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2102492733552726016","occurred_at":"2026-09-22T19:50:05Z","failed_at":"e2e","signature_id":"4fe90f7c3f08ac9dfbdbb935dc163323197da02f18ba46153bdcfc7eba9b8d77","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000914940\u003e: \n failed to create HCP cluster ea-list7cctv9: failed waiting for cluster=\"ea-list7cctv9\" in resourcegroup=\"ea-list-wc6fbxsd9rxx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3a3e500-1ba1-4cb7-8434-bc1aa96ee070\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3a3e500-1ba1-4cb7-8434-bc1aa96ee070\",\n \"name\": \"c3a3e500-1ba1-4cb7-8434-bc1aa96ee070\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T21:22:56.593640254Z\",\n \"endTime\": \"2026-09-22T21:51:58.372098632Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 2m51s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000914940\u003e: \n failed to create HCP cluster ea-list7cctv9: failed waiting for cluster=\"ea-list7cctv9\" in resourcegroup=\"ea-list-wc6fbxsd9rxx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3a3e500-1ba1-4cb7-8434-bc1aa96ee070\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3a3e500-1ba1-4cb7-8434-bc1aa96ee070\",\n \"name\": \"c3a3e500-1ba1-4cb7-8434-bc1aa96ee070\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T21:22:56.593640254Z\",\n \"endTime\": \"2026-09-22T21:51:58.372098632Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 2m51s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"38eeaeccffa0491180dfc6054cc1b3abf572643c67dff782fd1fdc281074fb46","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["351f45356c048233596e7dd87add23f59e5ae9c144a3196c0577133fac0ad667","38eeaeccffa0491180dfc6054cc1b3abf572643c67dff782fd1fdc281074fb46"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2102433878781202432","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-22T16:24:26Z","started_at":"2026-09-22T16:24:26Z","completed_at":"2026-09-22T18:37:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T18:49:38.328034058Z","timing_metadata_checked_at":"2026-09-22T18:49:38.328034058Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T18:49:38.333950039Z","region_metadata_checked_at":"2026-09-22T18:49:38.333950039Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2102385277220687872","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-22T13:11:15Z","started_at":"2026-09-22T13:11:15Z","completed_at":"2026-09-22T15:39:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T15:50:29.799605557Z","timing_metadata_checked_at":"2026-09-22T15:50:29.799605557Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T15:50:29.802526848Z","region_metadata_checked_at":"2026-09-22T15:50:29.802526848Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]},{"environment":"prod","summary":{"total_runs":2,"failed_runs":1,"runs_with_occurrences":1,"runs_with_failure_pattern_matches":1,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":1,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T23:00:41Z","started_at":"2026-09-22T23:00:41Z","completed_at":"2026-09-23T01:07:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T01:20:03.332552963Z","timing_metadata_checked_at":"2026-09-23T01:20:03.332552963Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T01:20:03.335451553Z","region_metadata_checked_at":"2026-09-23T01:20:03.335451553Z"},"failed_at":["e2e"],"failed_test_count":14,"occurrences":[{"row_id":"195ec1333def64e09e2e61fd4966d0631b09fb0bcf2805e3068b990e10572e14","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"d60ff50f583519824d6d7e580f49b3097c053eb851455b216fd32931071a2b74","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a7ee40\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-tcgbnw\" in resourcegroup=\"rg-np-version-upgrade-tcgbnw-hj2k4f59tdf7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-tcgbnw-hj2k4f59tdf7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-tcgbnw/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a7ee40\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-tcgbnw\" in resourcegroup=\"rg-np-version-upgrade-tcgbnw-hj2k4f59tdf7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-tcgbnw-hj2k4f59tdf7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-tcgbnw/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"1a44c4ed689ef8f4ee3fb01b65af6daf99c9c5881c465d13a54fa60f7c57e611","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"2d6b9a10fa16a43e4443cbb8ae50d90afa7db260b400159e32bd31c9c06e8fa6","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:274]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002790330\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-rnbkplrwl7s9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:274]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002790330\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-rnbkplrwl7s9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"489ee6c40ae76625b0679460ad64a7c1ae98a198dd436451bb9f198a4ef1ecf2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"dcdb32a276b85d5762c8f9a4390b250e8e8c154b174fce2166a1284975070064","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:202]: cluster CA data must contain exactly one PEM block\nExpected\n \u003c[]uint8 | len:6010, cap:6012\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:202]: cluster CA data must contain exactly one PEM block\nExpected\n \u003c[]uint8 | len:6010, cap:6012\u003e: ...\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1c0eb3626837a0d57ffaef6abf34daac6ebcb26dfffc58a3917d8718d8239c35","failure_pattern":"cluster CA data must contain exactly one PEM block","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:202]: cluster CA data must contain exactly one PEM block"}},{"row_id":"4e36f9469bc8016476fbc2cf2e6bb97297c95fce1a9836d0e222cc574034e30d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"93b07e54f382d1b86bb934122cf1eae24d4dca6e7c4ffc2ab45db781b7040196","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002942ed0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-khkzzgtsqgjz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002942ed0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-khkzzgtsqgjz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"5b20c93b88251314bc30b3b18cddff0ae98f3b965a94fd2d17caf69db06ddb08","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"67635489ff4055145750e643ed97308c677e61e4b811387b55d6f1c1a7f6d2c8","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:156]: failed to create node pool np-1 for TLS endpoint cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ef1980\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-dkldtr9wz7m4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/baff5654-9ec0-4bc5-a8a1-ffc26382827a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/baff5654-9ec0-4bc5-a8a1-ffc26382827a\",\n \"name\": \"baff5654-9ec0-4bc5-a8a1-ffc26382827a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T23:22:56.369741564Z\",\n \"endTime\": \"2026-09-22T23:42:02.437458738Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:156]: failed to create node pool np-1 for TLS endpoint cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ef1980\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-dkldtr9wz7m4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/baff5654-9ec0-4bc5-a8a1-ffc26382827a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/baff5654-9ec0-4bc5-a8a1-ffc26382827a\",\n \"name\": \"baff5654-9ec0-4bc5-a8a1-ffc26382827a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T23:22:56.369741564Z\",\n \"endTime\": \"2026-09-22T23:42:02.437458738Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5f250646b4ce6ddcf4ec00ae4d19f824fcc62e2c62258d589379f92861f43615","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"69994bd3122cfdf5cead21810ad001a53d6fca7838b640c3b6a6fcf65c3e702e","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0025133b0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-cxzbgzk6bm77/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0025133b0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-cxzbgzk6bm77/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"656f127f7b2c272e2643bcb68e16db4f1ca6f8fe0cb4d4f437024985b7b22c01","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"8165a74a386574cd51eb628b8ac81aa0881833bc8abe946100d3ecc127fe774a","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001002060\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-cqnjd26wmjcd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001002060\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-cqnjd26wmjcd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"6e188257e613a2b16bffe811ca2db4c6b0649ff48bd1af9b939fc215af3f8c12","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"a152d3d30eea393d48798792d545aabeaefde64210923bcf1f0905915e39788e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-6gs84k\" for cluster \"cluster-nightly-4-20-6gs84k\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce4120\u003e: \n failed to create NodePool np-6gs84k: failed starting nodepool creation \"np-6gs84k\" for cluster \"cluster-nightly-4-20-6gs84k\" in resourcegroup=\"rg-nightly-4-20-cv5qj2kl686k\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-cv5qj2kl686k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-6gs84k/nodePools/np-6gs84k\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-6gs84k\" for cluster \"cluster-nightly-4-20-6gs84k\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce4120\u003e: \n failed to create NodePool np-6gs84k: failed starting nodepool creation \"np-6gs84k\" for cluster \"cluster-nightly-4-20-6gs84k\" in resourcegroup=\"rg-nightly-4-20-cv5qj2kl686k\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-cv5qj2kl686k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-6gs84k/nodePools/np-6gs84k\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"82404c6c80507cbfca21bd6faa3dcde4183a34916b74ddf496e05d20257d9821","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"45414bb4c5c8a5a0bc7c8f0b4b63112f040f98363e403c86a8c64729cad56efa","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010981a0\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-x28k9h\" in resourcegroup=\"rg-np-skip-minor-x28k9h-x2cmwhr29p2s\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-x28k9h-x2cmwhr29p2s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-x28k9h/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010981a0\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-x28k9h\" in resourcegroup=\"rg-np-skip-minor-x28k9h-x2cmwhr29p2s\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-x28k9h-x2cmwhr29p2s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-x28k9h/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"9ad0c675730ea425ab5ddb0b7c241208809fbc5dfbcbcf3b38631afe64a12d79","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"589aee349dfeeb917b1e915467908bfe001aded4ad5fc993461f2c4f097923d5","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00148e5d0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-7dfqps65dplq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00148e5d0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-7dfqps65dplq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"9ca986d95db96b4d4fec0af343f4072449e2aea18ccfa6957ed17462ef6d3de2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"32e3a94a639de7d26ad7cb7a82730cf1fe540c3bacab33ab7a4b65d652194ad6","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-5-1-ctwh8htmlpcs/cluster-nightly-5-1-v48g4k should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00150a700\u003e: \n failed to create HCP cluster cluster-nightly-5-1-v48g4k: failed waiting for cluster=\"cluster-nightly-5-1-v48g4k\" in resourcegroup=\"rg-nightly-5-1-ctwh8htmlpcs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/219d6b96-27bd-42d1-8102-d400db0e0bb5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/219d6b96-27bd-42d1-8102-d400db0e0bb5\",\n \"name\": \"219d6b96-27bd-42d1-8102-d400db0e0bb5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T23:38:52.209443302Z\",\n \"endTime\": \"2026-09-23T00:07:57.16268573Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-5-1-ctwh8htmlpcs/cluster-nightly-5-1-v48g4k should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00150a700\u003e: \n failed to create HCP cluster cluster-nightly-5-1-v48g4k: failed waiting for cluster=\"cluster-nightly-5-1-v48g4k\" in resourcegroup=\"rg-nightly-5-1-ctwh8htmlpcs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/219d6b96-27bd-42d1-8102-d400db0e0bb5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/219d6b96-27bd-42d1-8102-d400db0e0bb5\",\n \"name\": \"219d6b96-27bd-42d1-8102-d400db0e0bb5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-22T23:38:52.209443302Z\",\n \"endTime\": \"2026-09-23T00:07:57.16268573Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b70ab4d02af0909c518667d9c11e7403ccb6eb18215ba62cbeebff2248acfb28","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"7756282e4c2797adc5a9559f0aecc68f5113799a329e53f6fead9f7607a3e463","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00103f360\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-xqlnpq2wtgvl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-xqlnpq2wtgvl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00103f360\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-xqlnpq2wtgvl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-xqlnpq2wtgvl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d6164cff5eccd3db4fabd19fbe08aa12f39c6b3d5c2202182c9f291029d16282","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.22\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"ef8ac65f7970decf85544fc46b904e5252cd41d3c85dab26642d449a03e0efee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"0cfcc19325c94c6fcd7928e1960873ccb847383b907d9a4008c9f2eef815cedb","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.006s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000fb5908\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-21-210852, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.006s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000fb5908\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-21-210852, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"ffa55cdd2435703093ebcdd159d915495bd381ec51da11946976b4eda97e0641","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"2553e871dbbe3bcfe30dc281f555126453c37b319bd95a2ea76f0bd2cd8c9c80","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001385da0\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-4tjrbb\" resourcegroup=\"rg-np-dg-minor-4tjrbb-jd2b4bxb9q5x\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-4tjrbb-jd2b4bxb9q5x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-4tjrbb/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001385da0\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-4tjrbb\" resourcegroup=\"rg-np-dg-minor-4tjrbb-jd2b4bxb9q5x\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-4tjrbb-jd2b4bxb9q5x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-4tjrbb/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":14,"failure_pattern_ids":["1c0eb3626837a0d57ffaef6abf34daac6ebcb26dfffc58a3917d8718d8239c35","33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","d6164cff5eccd3db4fabd19fbe08aa12f39c6b3d5c2202182c9f291029d16282"],"matched_occurrences":14,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102216478647390208","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-22T02:00:40Z","started_at":"2026-09-22T02:00:40Z","completed_at":"2026-09-22T04:10:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T04:22:53.704082977Z","timing_metadata_checked_at":"2026-09-22T04:22:53.704082977Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T04:22:53.707935664Z","region_metadata_checked_at":"2026-09-22T04:22:53.707935664Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]},{"environment":"stg","summary":{"total_runs":2,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T23:00:41Z","started_at":"2026-09-22T23:00:41Z","completed_at":"2026-09-22T23:16:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T23:23:51.428007926Z","timing_metadata_checked_at":"2026-09-22T23:23:51.428007926Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T23:23:51.430723016Z","region_metadata_checked_at":"2026-09-22T23:23:51.430723016Z"},"failed_at":["e2e"],"failed_test_count":52,"occurrences":[{"row_id":"00b7a3b644f2b2c2b2feb930b305264a5ebae3852c69a2889832682ed2fb14d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"b6c4f658fa4d8951a503f16688a96dec150a63ee3bae940202661178db6f28df","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001022320\u003e: \n failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-5c69lbjzcdrv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-5c69lbjzcdrv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001022320\u003e: \n failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-5c69lbjzcdrv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-5c69lbjzcdrv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"082169bff2b27f83ee12a6c778381b307f2ce4c447a68a984cd4110a8dd35a1c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"9f14e27a49ec3c045e8184698464b340e779ebc18fed8b6d564892bad88dc4d1","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-9r6pb2\" at install version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000618600\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-9r6pb2: failed starting cluster creation \"cp-ystream-upgrade-4-21-9r6pb2\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-8wrqg8zdkgps\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-8wrqg8zdkgps/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-9r6pb2\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-9r6pb2\" at install version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000618600\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-9r6pb2: failed starting cluster creation \"cp-ystream-upgrade-4-21-9r6pb2\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-8wrqg8zdkgps\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-8wrqg8zdkgps/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-9r6pb2\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"08f9225cdb735fc2f569fbf85df61e4c2f52b60008c4028db47098c9abb5cde3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"71fdaf900552131812fcae8b61a9676607004ea56053f4c3d2f760b74c226ad3","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001597a40\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001597a40\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"0b6750c8ea36eb046ab1bee1e0d993aa82cbc25a190f152ce33dd354396eaf76","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"9064132d524eca5a110d383b68965c649423f2c6a7a98c011daa34d30091d1fd","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f88140\u003e: \n failed starting cluster creation \"agg-cluster\" in resourcegroup=\"feature-aggregation-8vpkwbcpkrmb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-8vpkwbcpkrmb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f88140\u003e: \n failed starting cluster creation \"agg-cluster\" in resourcegroup=\"feature-aggregation-8vpkwbcpkrmb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-8vpkwbcpkrmb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"0c0e08d4ddb7bdd213496df7b5129aff34f631018a0e0ce4909442f46e15a86b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"308bccdf6612e595b3ab97a427e5b7cf09775b60ed9f62490094fa231150246e","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001782240\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed starting cluster creation \"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-2xjmn786m79q\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-2xjmn786m79q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/missing-mi-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001782240\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed starting cluster creation \"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-2xjmn786m79q\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-2xjmn786m79q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/missing-mi-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"120e3e45a1d333547c7bd13c186ceecb347b0669f8e63e4eb68e2de1d9e5e23c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"1f815aa53ea4d4346365db14604b88f15bc2ed059f664fb14b4af3f3ca4e528c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-20-5jxwkvlcphv4/cluster-nightly-4-20-59hchb should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010903c0\u003e: \n failed to create HCP cluster cluster-nightly-4-20-59hchb: failed starting cluster creation \"cluster-nightly-4-20-59hchb\" in resourcegroup=\"rg-nightly-4-20-5jxwkvlcphv4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-5jxwkvlcphv4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-59hchb\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-20-5jxwkvlcphv4/cluster-nightly-4-20-59hchb should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010903c0\u003e: \n failed to create HCP cluster cluster-nightly-4-20-59hchb: failed starting cluster creation \"cluster-nightly-4-20-59hchb\" in resourcegroup=\"rg-nightly-4-20-5jxwkvlcphv4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-5jxwkvlcphv4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-59hchb\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"13867b89e60b04ddaff6be93e7e68c8abbd1c9125512e9265bf8fd45ccd200b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"fd166ea65a29cb24c93f31900728e61a500d3e97d9d71b119d88d8174ed940da","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-mtz6w6 at version 4.22.0-0.nightly-multi-2026-09-17-071442\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bf4220\u003e: \n failed to create HCP cluster np-skip-minor-mtz6w6: failed starting cluster creation \"np-skip-minor-mtz6w6\" in resourcegroup=\"rg-np-skip-minor-mtz6w6-kplfq87nckfq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-mtz6w6-kplfq87nckfq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-mtz6w6\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-mtz6w6 at version 4.22.0-0.nightly-multi-2026-09-17-071442\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bf4220\u003e: \n failed to create HCP cluster np-skip-minor-mtz6w6: failed starting cluster creation \"np-skip-minor-mtz6w6\" in resourcegroup=\"rg-np-skip-minor-mtz6w6-kplfq87nckfq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-mtz6w6-kplfq87nckfq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-mtz6w6\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"187265c629f40ba7358e7fe6bd32b9d3dfd31576fc2d27f69eb5067c98681902","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"679c9425ef46695adbfae3247fef9b32a62432dc8b71340ca0632dfc151535c1","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a562c0\u003e: \n failed to create HCP cluster ea-listt8rz8v: failed starting cluster creation \"ea-listt8rz8v\" in resourcegroup=\"ea-list-bvk5zpjkh2nd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-bvk5zpjkh2nd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listt8rz8v\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a562c0\u003e: \n failed to create HCP cluster ea-listt8rz8v: failed starting cluster creation \"ea-listt8rz8v\" in resourcegroup=\"ea-list-bvk5zpjkh2nd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-bvk5zpjkh2nd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listt8rz8v\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"1b71e81969725c5c51f846f3589c7cd56d70fae3b150347a1c99ad6785a541f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"e0ab066f324eff7b3bcbcc6c36d44358b76341a4409a3592b7bd7df88172d18d","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-nn5gnm\" at install version 4.22.0-0.nightly-multi-2026-09-17-071442\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b5a0e0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-nn5gnm: failed starting cluster creation \"cp-ystream-upgrade-5-0-nn5gnm\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-cn4pm599hjxc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-cn4pm599hjxc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-nn5gnm\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-nn5gnm\" at install version 4.22.0-0.nightly-multi-2026-09-17-071442\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b5a0e0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-nn5gnm: failed starting cluster creation \"cp-ystream-upgrade-5-0-nn5gnm\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-cn4pm599hjxc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-cn4pm599hjxc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-nn5gnm\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"2027446d77fd2a2f2b2f862096f428da33d2a1097167c35ab424bbf026aa9d2f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"158f12004ca1915f2f3191b52a2e9cf87648c5b229999fc1a8b981b548f5d2ab","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-5-0-8kk449x5s6qc/cluster-nightly-5-0-bhqwr9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e4e280\u003e: \n failed to create HCP cluster cluster-nightly-5-0-bhqwr9: failed starting cluster creation \"cluster-nightly-5-0-bhqwr9\" in resourcegroup=\"rg-nightly-5-0-8kk449x5s6qc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-8kk449x5s6qc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-5-0-bhqwr9\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-5-0-8kk449x5s6qc/cluster-nightly-5-0-bhqwr9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e4e280\u003e: \n failed to create HCP cluster cluster-nightly-5-0-bhqwr9: failed starting cluster creation \"cluster-nightly-5-0-bhqwr9\" in resourcegroup=\"rg-nightly-5-0-8kk449x5s6qc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-8kk449x5s6qc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-5-0-bhqwr9\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"2258907f06a06e3db0e8b96a57b7558f640f4dc03d76abe130742c6c77d0239c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"1455b7a170049f52e308cbcd54385fa67ff255e0e6641c47fd37f7755b51636c","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101bd80\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-jnltm5nznwcw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-jnltm5nznwcw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101bd80\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-jnltm5nznwcw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-jnltm5nznwcw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"2c4ba13d2596de714e868a1432216476851ff49a56751ad245f2036ba3e7ff72","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"028e67617c89905a105cc4a612840e75f6897bb9addb98ab7bb40d8a55372dc6","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000698d00\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-9ggtk5tbv4fl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-9ggtk5tbv4fl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000698d00\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-9ggtk5tbv4fl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-9ggtk5tbv4fl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"2da9ae30a30785341b52b658b182388b40dc7b9a734fd810b4df69bc7555bc7f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"f165c16c6c0a25cc2f95a20d410a0506031f94afecbdcbe9fcb801885d325f60","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001592140\u003e: \n failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-xc8jsrhsbqgs\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-xc8jsrhsbqgs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001592140\u003e: \n failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-xc8jsrhsbqgs\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-xc8jsrhsbqgs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"3489ae9527f572caf259600e5b5ff0a000e269f034368c53908f4a448946bfb4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"a83dc4016d9d981f2ca25c792ccf0d882b5f2b518ba93befcd305baa0e72a752","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011ba60\u003e: \n failed to create HCP cluster np-dg-minor-hgh59k: failed starting cluster creation \"np-dg-minor-hgh59k\" in resourcegroup=\"rg-np-dg-minor-hgh59k-49x99ndztwrj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-hgh59k-49x99ndztwrj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-hgh59k\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011ba60\u003e: \n failed to create HCP cluster np-dg-minor-hgh59k: failed starting cluster creation \"np-dg-minor-hgh59k\" in resourcegroup=\"rg-np-dg-minor-hgh59k-49x99ndztwrj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-hgh59k-49x99ndztwrj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-hgh59k\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"35bc8e6e5023bcc8c4491370d8752d3b95313d103fabbf8c2dca81ac01228efa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"df55f6bed89a1af8c0cc697d8652b30a0d8b6414f6f83f78d0039e476d8851be","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00096a120\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed starting cluster creation \"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-dw22pxrnfnk6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-dw22pxrnfnk6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00096a120\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed starting cluster creation \"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-dw22pxrnfnk6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-dw22pxrnfnk6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"39038e03cef344b477ff14e0f27e1a772b16dab30c0f24ecc74d8dc2d5ca9a37","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"b9b567307ed244c17f853e374b1c78b752df4906beb3ff703c0d2473f1b6fbcd","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b90100\u003e: \n failed to create HCP cluster ea-cluster: failed starting cluster creation \"ea-cluster\" in resourcegroup=\"external-auth-cluster-g86rpg6nvsxm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-g86rpg6nvsxm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b90100\u003e: \n failed to create HCP cluster ea-cluster: failed starting cluster creation \"ea-cluster\" in resourcegroup=\"external-auth-cluster-g86rpg6nvsxm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-g86rpg6nvsxm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"390f7d335242ffc51d78dd2472486825ac3ea8970e98844caf430739dc4e24ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"95cd434bfdd2f7a22f7b502fbf40a0223dcd791fe2fe4e9d8441d5a79f1e9858","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001180240\u003e: \n failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-hbfdw9rw98ps\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-hbfdw9rw98ps/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001180240\u003e: \n failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-hbfdw9rw98ps\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-hbfdw9rw98ps/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"39b2f568715ea9aa529d20a58f10da333aed9db71bf618add988d331bcc0d39d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"87c4328596602d9fee11e1d5586b16d36dfdea3ecc244b5be95e0ead9e07fe5b","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c92540\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-dr6j2dvf9kv5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-dr6j2dvf9kv5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c92540\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-dr6j2dvf9kv5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-dr6j2dvf9kv5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"39f283c234a5ad0e907b8aa6caceb37af1d4817d05d649b62ba5b354de6bc720","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"e1c82124e155c1115d2dbcf9a50d4bc2c94263abc4b30689eb04b100350c6c30","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa4120\u003e: \n failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-x4n5jq88swcc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-x4n5jq88swcc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa4120\u003e: \n failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-x4n5jq88swcc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-x4n5jq88swcc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"3b0209d872ad36b042679b74c2e6f13a469ed4a132a46845e3d74a773aed642d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"a9516bb510596b90539cbfadae884a6fc2ee3e66cfb81c22faeb69b9fccd6e4b","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000618180\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed starting cluster creation \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-5g9nckwfbg5h\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-5g9nckwfbg5h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-delete-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000618180\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed starting cluster creation \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-5g9nckwfbg5h\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-5g9nckwfbg5h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-delete-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"400721f3df6a46ea14c44cb4cf0053c1a923e70b8eac4c10151ab59d1595e85e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"e4f7cef6331c46e4fab04aabb85570a54c299a0ee51f44135e8f8b3cad753c3f","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fd6100\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-nnlw6xnxb26v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-nnlw6xnxb26v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fd6100\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-nnlw6xnxb26v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-nnlw6xnxb26v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"4517f650e1189c654a6f334bedb83d5e337008b93c160483844c25d2860e7349","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"fef37f6e3d1c779f49824962c264c15d6c4f20dc311cabef868b4444b33225d4","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-sctgzn\" at install version 4.22.0-0.nightly-multi-2026-09-17-071442\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d7a1a0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-sctgzn: failed starting cluster creation \"cp-ystream-upgrade-4-23-sctgzn\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-vdl85ff8lc86\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-vdl85ff8lc86/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-sctgzn\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-sctgzn\" at install version 4.22.0-0.nightly-multi-2026-09-17-071442\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d7a1a0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-sctgzn: failed starting cluster creation \"cp-ystream-upgrade-4-23-sctgzn\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-vdl85ff8lc86\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-vdl85ff8lc86/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-sctgzn\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"4c8cd3ad8e0cdb3b960e58d9de124cab53e4c0e934ccb05880a9a9a11a8c1db4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"4d9b7f2662dcb7745409a8437cd36bb2b7fe3d3744f2311af178529ff0180473","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012a63e0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-ms56qpvz7f4t\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-ms56qpvz7f4t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012a63e0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-ms56qpvz7f4t\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-ms56qpvz7f4t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"4ccf1c1a68511dade873fcd0cee433794bb805054f5f04126554df8bf0edf5f1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"15b911e33e9c755011628b5fe192da1356eb0d4cb047227a8bc58def8652e545","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:103]: failed to begin creating HCP cluster \"admin-cred-lifecycle-2r66p6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000abad00\u003e: \n failed starting cluster creation \"admin-cred-lifecycle-2r66p6\" in resourcegroup=\"admin-credential-lifecycle-test-5fm57kncmfth\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-5fm57kncmfth/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-2r66p6\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:103]: failed to begin creating HCP cluster \"admin-cred-lifecycle-2r66p6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000abad00\u003e: \n failed starting cluster creation \"admin-cred-lifecycle-2r66p6\" in resourcegroup=\"admin-credential-lifecycle-test-5fm57kncmfth\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-5fm57kncmfth/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-2r66p6\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"53c8132912d0ee9cf4a98f728903c3fbf14d51fa0abd22aef6872149a2f8482c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"a075e6218bb7744e36b44936eb254dff2e82cba2a27dbf09d70bf39a3c405d3b","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d86180\u003e: \n failed starting cluster creation \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-9h8ggzx7ljbh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-9h8ggzx7ljbh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-20\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d86180\u003e: \n failed starting cluster creation \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-9h8ggzx7ljbh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-9h8ggzx7ljbh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-20\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"56d9d01fd1194816c371fd12d2b857c95b596b406bf02a1fc1077c958501edbf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"f39ebe4a0a487af20c13da8463b9fdeddfc15918af2a9bf235cac92bc6528e5f","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001b18840\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-ntrplhzxjq86/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001b18840\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-ntrplhzxjq86/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"5c1af90df598b0ca9c3ff1600ef0c9c9bbc207d8adf0b10fcbc671262917cd17","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"6fe2156bf8cfb75c6d8f9c2fee4e593a246239df1895f046cb9af18cfcfea086","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005ae660\u003e: \n failed to create HCP cluster negative-tests-cluster: failed starting cluster creation \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-8zk685kvqsgq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-8zk685kvqsgq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005ae660\u003e: \n failed to create HCP cluster negative-tests-cluster: failed starting cluster creation \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-8zk685kvqsgq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-8zk685kvqsgq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"604f89048351a39bc87ac2c70bfbb7cee5bbe52cee55162b2727634350174653","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"74f3a750328c3cd542cbb06e73e38642e88e91149b817dbcf78144f7db362c36","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-5-1-7m29nrv4fdr6/cluster-nightly-5-1-scx85f should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013fc280\u003e: \n failed to create HCP cluster cluster-nightly-5-1-scx85f: failed starting cluster creation \"cluster-nightly-5-1-scx85f\" in resourcegroup=\"rg-nightly-5-1-7m29nrv4fdr6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-1-7m29nrv4fdr6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-5-1-scx85f\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-5-1-7m29nrv4fdr6/cluster-nightly-5-1-scx85f should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013fc280\u003e: \n failed to create HCP cluster cluster-nightly-5-1-scx85f: failed starting cluster creation \"cluster-nightly-5-1-scx85f\" in resourcegroup=\"rg-nightly-5-1-7m29nrv4fdr6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-1-7m29nrv4fdr6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-5-1-scx85f\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6653e7c70806e8600f2c594010955907fa511ae13bd5b39868e53b8fbf98e8ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"5eb660c4b9631ed92df772ec5d0f274a8e95adf3b7ce117f39890c1707545c26","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001030240\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-42c2h57lg25f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-42c2h57lg25f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001030240\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-42c2h57lg25f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-42c2h57lg25f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6eef33b7f68bb510e857ddeffc7634c68cbe04595d12de1c733250e1e5935358","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"8f31a14086c93cf05fa58efd6f7ac6f6792c564a3a78418503408e590ce4458c","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000682260\u003e: \n failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-j585nqsnf67j\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-j585nqsnf67j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000682260\u003e: \n failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-j585nqsnf67j\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-j585nqsnf67j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7ff2fb75c9bbd5a3a154ef23672a5c6114b71457b5c698994a7c3ae1c5b11c86","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"a22cdef3c94a4f22d3948d36021a322a8dc677144620cb5969989b081354c899","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-c4n9pr with version 4.21.0-0.nightly-multi-2026-09-22-123437\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bb8780\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-c4n9pr: failed starting cluster creation \"np-version-upgrade-cluster-c4n9pr\" in resourcegroup=\"rg-np-version-upgrade-c4n9pr-lvc25kgsdccb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-c4n9pr-lvc25kgsdccb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-c4n9pr\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-c4n9pr with version 4.21.0-0.nightly-multi-2026-09-22-123437\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bb8780\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-c4n9pr: failed starting cluster creation \"np-version-upgrade-cluster-c4n9pr\" in resourcegroup=\"rg-np-version-upgrade-c4n9pr-lvc25kgsdccb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-c4n9pr-lvc25kgsdccb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-c4n9pr\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"802ffc4658ce38215cc049347fa86b1d066e2a11782183ea8f868d75acd621fb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"14a8c56985cacc9ae125bee7248507f8c5ec554db604d66afe031c0038c6c3de","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-22-r8dcl75vwsvm/cluster-nightly-4-22-rbw68x should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000714200\u003e: \n failed to create HCP cluster cluster-nightly-4-22-rbw68x: failed starting cluster creation \"cluster-nightly-4-22-rbw68x\" in resourcegroup=\"rg-nightly-4-22-r8dcl75vwsvm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-r8dcl75vwsvm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-22-rbw68x\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-22-r8dcl75vwsvm/cluster-nightly-4-22-rbw68x should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000714200\u003e: \n failed to create HCP cluster cluster-nightly-4-22-rbw68x: failed starting cluster creation \"cluster-nightly-4-22-rbw68x\" in resourcegroup=\"rg-nightly-4-22-r8dcl75vwsvm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-r8dcl75vwsvm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-22-rbw68x\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"8a1b15f67e1f7eb26a1e1e0946e3cd828caf6ffabac136c61b8fa4c34d688ab4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"ee4aafc530fdc6d6b01c26778a1358106dabb25097f0564cca2e72c97f9d0658","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011b8a0\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed starting cluster creation \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-877p4644xp66\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-877p4644xp66/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011b8a0\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed starting cluster creation \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-877p4644xp66\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-877p4644xp66/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"9fc1c199b5f3eecafffa97b3a9e0928a821c4a387758ce7a362b5bcbc948e385","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"200a19ebdb17256d9331c2aa684172e1327200af8b213c83c2417edad907a31d","test_name":"Customer should be able to update node pool labels and taints","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aec280\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed starting cluster creation \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-ggdtclvwvh49\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-ggdtclvwvh49/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aec280\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed starting cluster creation \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-ggdtclvwvh49\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-ggdtclvwvh49/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c35c97ae266d42b7115a9df918302f4eb6816bc9b0924cb29a66a299fd81b965","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"caefb865d0b3a4db06386013abdc426cf7ade03892faf311d759877af93b2b20","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001b3020\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-x64bd6kgw77d\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-x64bd6kgw77d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001b3020\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-x64bd6kgw77d\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-x64bd6kgw77d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c454dafd121ce18c767706c2d9ff7f63c905db30cbea9d133f62b52ac36ed1ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"b6092b5834862dcdbff59994ecf5c15516918216b25624a1a8e34e8cf20ab91f","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e98240\u003e: \n failed starting cluster creation \"private-kv-cluster\" in resourcegroup=\"private-keyvault-nkc9f66h7mrb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-nkc9f66h7mrb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e98240\u003e: \n failed starting cluster creation \"private-kv-cluster\" in resourcegroup=\"private-keyvault-nkc9f66h7mrb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-nkc9f66h7mrb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d022af785fa95f7e706c64a46530d2d7aed28947132364883d18ab2d8654b413","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"5001ea95a48aa452365fad4a89be1585157725813537a20c155d45169398c028","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00140c260\u003e: \n failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-bmczqgpdvhmg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-bmczqgpdvhmg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00140c260\u003e: \n failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-bmczqgpdvhmg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-bmczqgpdvhmg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d39f474c0936c163c77605d2a7c985a3086ab47ac9f392669b7ce5237b6cd5ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"f644f319f19d576779a8299466637a992ed29ad34884b8a2b8b76112b6143919","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010740e0\u003e: \n failed to create HCP cluster cilium-cl: failed starting cluster creation \"cilium-cl\" in resourcegroup=\"cni-cilium-snrfdxp7lz6k\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-snrfdxp7lz6k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cl\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010740e0\u003e: \n failed to create HCP cluster cilium-cl: failed starting cluster creation \"cilium-cl\" in resourcegroup=\"cni-cilium-snrfdxp7lz6k\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-snrfdxp7lz6k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cl\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d5bbfe4a08f125dd15ecd75b6dc91e666f30770aed08aa30ccd3a3a7f7e1dc50","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"e4f0530149d9c515a8bb2e10bea920e06291e06a625c67e64d57fa1abd5a5b74","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000624460\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-72hqfj82n768\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-72hqfj82n768/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000624460\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-72hqfj82n768\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-72hqfj82n768/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d94578f1a1500f1d30747468539bc7382bd7c1724094229572785c0a12c35c8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"af3f5db7f3b8aab06ccf785bc8dcf9812f403ee46c33dd1f94132db968f544ac","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00114e160\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-bv97895wknf4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-bv97895wknf4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00114e160\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-bv97895wknf4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-bv97895wknf4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"daa61c3f2cbd4079c3775725fc52467a8ca5fec19d660bba8f46b6474dcd71c4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"b06c1efb61f84c865ba6cce98104579878135b2fcb639431f48485564f5334b9","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c82a40\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-b6pmc9sdknkf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-b6pmc9sdknkf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c82a40\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-b6pmc9sdknkf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-b6pmc9sdknkf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"df53e4a3d5feca9cc98a3e713985537f60235948d67c3327d9664c038c3d1901","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"c05362ff9e4d0172e5ae29c0bfaffb620c3a19b99e013eacfd7f50ea0a619126","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013160e0\u003e: \n failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-rx9wkcqgtnhz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-rx9wkcqgtnhz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013160e0\u003e: \n failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-rx9wkcqgtnhz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-rx9wkcqgtnhz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"e087cbb4e0b3e7f89219724f0face31681fee480fb5fe5c80a8178960e5dca34","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"1659d60c3f4e509c612345cd89191604a251a0a8078810794c41b4c4cdea090d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-23-bzpzlqbzxqvl/cluster-nightly-4-23-p7gkv7 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009be140\u003e: \n failed to create HCP cluster cluster-nightly-4-23-p7gkv7: failed starting cluster creation \"cluster-nightly-4-23-p7gkv7\" in resourcegroup=\"rg-nightly-4-23-bzpzlqbzxqvl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-23-bzpzlqbzxqvl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-23-p7gkv7\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-23-bzpzlqbzxqvl/cluster-nightly-4-23-p7gkv7 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009be140\u003e: \n failed to create HCP cluster cluster-nightly-4-23-p7gkv7: failed starting cluster creation \"cluster-nightly-4-23-p7gkv7\" in resourcegroup=\"rg-nightly-4-23-bzpzlqbzxqvl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-23-bzpzlqbzxqvl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-23-p7gkv7\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"e4b2625153d31ff778489d83c76a346cb122aabcedca584de45583959bd79931","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"810cbf6d7f7e2ac90d52b67a649ab8562d7f958a52dca5ad11a67a640ebe8cea","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000886420\u003e: \n failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-ps8hpf8pd6z9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-ps8hpf8pd6z9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000886420\u003e: \n failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-ps8hpf8pd6z9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-ps8hpf8pd6z9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ea93bba8c3b7b97de77cabdcf3c6f75d020ebb9f39f0d07fc0fe5fd15657fc1b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"559b8323654a159af3325d41df0b8f67a4560c8100c549c7010720e017f0392e","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-frkjxj\" at install version 5.0.0-0.nightly-multi-2026-09-21-210852\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bfe2e0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-1-frkjxj: failed starting cluster creation \"cp-ystream-upgrade-5-1-frkjxj\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-kcgn55zdv82f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-kcgn55zdv82f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-frkjxj\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-frkjxj\" at install version 5.0.0-0.nightly-multi-2026-09-21-210852\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bfe2e0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-1-frkjxj: failed starting cluster creation \"cp-ystream-upgrade-5-1-frkjxj\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-kcgn55zdv82f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-kcgn55zdv82f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-frkjxj\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"eac551ce8c8626eb83553c5a377107f80e1189ddf28c25ac38c086253c5f544d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"80ab27c2003ae673a88894b4f16c2b34be3d2886fa843507985c0ecf0615f388","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000744c40\u003e: \n failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \"patch-tags-cluster\" in resourcegroup=\"patch-tags-2jnwvdzzh64d\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-2jnwvdzzh64d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000744c40\u003e: \n failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \"patch-tags-cluster\" in resourcegroup=\"patch-tags-2jnwvdzzh64d\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-2jnwvdzzh64d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ed56b3850cf1e798b0a9fe77c29c6f8f6ede873d8484ff2f278cdf080280d553","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"c6f8e5571aa54819347dcd74c64010291cb3bbe12b83892aeb0d039233f5110e","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001072300\u003e: \n failed to create HCP cluster node-limit-cluster: failed starting cluster creation \"node-limit-cluster\" in resourcegroup=\"node-limit-test-gtbrw7nrxjz2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-gtbrw7nrxjz2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001072300\u003e: \n failed to create HCP cluster node-limit-cluster: failed starting cluster creation \"node-limit-cluster\" in resourcegroup=\"node-limit-test-gtbrw7nrxjz2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-gtbrw7nrxjz2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"eefe84a002aae4a7c5d22cc9532d74e7cb881c7ae01cc682cd08a2abf0c34396","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"d7399f90009945d8db59430f56c61cae27445036521760c591629a465c830d13","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-kx2v6b\" at install version 4.21.0-0.nightly-multi-2026-09-22-123437\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c83440\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-kx2v6b: failed starting cluster creation \"cp-ystream-upgrade-4-22-kx2v6b\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-sqqmsdwsqbrs\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-sqqmsdwsqbrs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-kx2v6b\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-kx2v6b\" at install version 4.21.0-0.nightly-multi-2026-09-22-123437\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c83440\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-kx2v6b: failed starting cluster creation \"cp-ystream-upgrade-4-22-kx2v6b\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-sqqmsdwsqbrs\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-sqqmsdwsqbrs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-kx2v6b\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f181160445bed9b00f78a7ef712bc413fe77aef25ca788e7666d67868fdc29b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"d507d7cf0f593f52cca6bf6f56c14130a86590f9713918391811fa4a6919b30c","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001052180\u003e: \n failed starting cluster creation \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-khqv7gn8cz69\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-khqv7gn8cz69/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001052180\u003e: \n failed starting cluster creation \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-khqv7gn8cz69\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-khqv7gn8cz69/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f726e297e8f37d2256fd997c0c2e0dd520a034387f7bc3e408dd4bf445be608d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"56b282587877a0c992745a7e777f42c2329ad8e8d2870c9b16dee50a29ecc3fa","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00102c3a0\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-4qb52bqkpzjf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-4qb52bqkpzjf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00102c3a0\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-4qb52bqkpzjf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-4qb52bqkpzjf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f8bd3dfd5fa14ed68998d98d63473d6146036cd36c06fbf586bb22a79afde4fb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"134078396d0d2e90e6acf179f67c0c5c29619d70cb5b99448169bae9bf7a53c2","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006784e0\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed starting cluster creation \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-l2fgd6zqtf9h\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-l2fgd6zqtf9h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cx-rg-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006784e0\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed starting cluster creation \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-l2fgd6zqtf9h\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-l2fgd6zqtf9h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cx-rg-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"fd283e268211d169c1bc7215386ad225c367a0cc42b97a0f4672cbb4e6893e3a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","failed_at":"e2e","signature_id":"a847f67769c58d34c5b1c9f5edb54c406bd4ce3097b77e36d148660e89714943","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-21-v8sz4sgjpztm/cluster-nightly-4-21-9lgzcb should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cb22a0\u003e: \n failed to create HCP cluster cluster-nightly-4-21-9lgzcb: failed starting cluster creation \"cluster-nightly-4-21-9lgzcb\" in resourcegroup=\"rg-nightly-4-21-v8sz4sgjpztm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-21-v8sz4sgjpztm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-21-9lgzcb\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-21-v8sz4sgjpztm/cluster-nightly-4-21-9lgzcb should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cb22a0\u003e: \n failed to create HCP cluster cluster-nightly-4-21-9lgzcb: failed starting cluster creation \"cluster-nightly-4-21-9lgzcb\" in resourcegroup=\"rg-nightly-4-21-v8sz4sgjpztm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-21-v8sz4sgjpztm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-21-9lgzcb\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}}],"failure_pattern_summary":{"signature_count":52,"failure_pattern_ids":["82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54"],"matched_occurrences":52,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-22T02:00:40Z","started_at":"2026-09-22T02:00:40Z","completed_at":"2026-09-22T02:16:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-22T02:18:06.810761575Z","timing_metadata_checked_at":"2026-09-22T02:18:06.810761575Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-22T02:18:06.813458267Z","region_metadata_checked_at":"2026-09-22T02:18:06.813458267Z"},"failed_at":["e2e"],"failed_test_count":56,"occurrences":[{"row_id":"02324bd4d582afd6982b04a6789856e8e1b2976b1e552107c2fb9d9249c54e55","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"35bdbace1efff0b1b9097d9b772f2e08e1c597b678932a37cc1b7e87d68c5529","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f300e0\u003e: \n failed to create HCP cluster node-limit-cluster: failed starting cluster creation \"node-limit-cluster\" in resourcegroup=\"node-limit-test-rz6r289t49jf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-rz6r289t49jf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f300e0\u003e: \n failed to create HCP cluster node-limit-cluster: failed starting cluster creation \"node-limit-cluster\" in resourcegroup=\"node-limit-test-rz6r289t49jf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-rz6r289t49jf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"0b715a1599c2a154748221019e1f15e946847026c2180c5c312f3058fd51c739","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"9e5b3990ed8a6ed539792fe22e7870b8cbfe2fc062043db588d0f769a6c0f78e","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b821e0\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-474zgvs2k728\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-474zgvs2k728/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b821e0\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-474zgvs2k728\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-474zgvs2k728/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"10073583c3743cf8d16af3d263ef8d20945e1950ed5bb30c33f60fdcf82f4ccf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"06d9be7494f12960fdbcec7b743d661d01c049e05bb9bba0d808e823945968f7","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-xbrtd7\" with version 5.0.0-rc.2 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b30480\u003e: \n failed to create HCP cluster cluster-zstream-5-0-xbrtd7: failed starting cluster creation \"cluster-zstream-5-0-xbrtd7\" in resourcegroup=\"rg-zstream-upgrade-5-0-w6cvb247pkgq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-w6cvb247pkgq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-xbrtd7\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-xbrtd7\" with version 5.0.0-rc.2 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b30480\u003e: \n failed to create HCP cluster cluster-zstream-5-0-xbrtd7: failed starting cluster creation \"cluster-zstream-5-0-xbrtd7\" in resourcegroup=\"rg-zstream-upgrade-5-0-w6cvb247pkgq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-w6cvb247pkgq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-xbrtd7\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"15d7c46a2ef2438cc209cd108a077d70f6477c509bd64d09ba6f73032426a901","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"b481b2d65ab6004bb5d2705a6a89dbd0e37b8346d88ed8d9125cb85ab5882086","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bcca20\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-zrxpwlk9jdzp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-zrxpwlk9jdzp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bcca20\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-zrxpwlk9jdzp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-zrxpwlk9jdzp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"18986b6de0c33138d3bdd7234d2022cffd84790e61f841cf436f2d4b654aecf5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"b118fb558f5f55539b11bc8c16f10ac542da724b4d40c700527fdb51e92324d0","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f1e0e0\u003e: \n failed starting cluster creation \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-8fc9fdnjqp2w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-8fc9fdnjqp2w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f1e0e0\u003e: \n failed starting cluster creation \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-8fc9fdnjqp2w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-8fc9fdnjqp2w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"1a340fa500c346d1e7eef8ce821991826d07920937dcdc421488529295d46609","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"99c18bdee1e1f06cea990db26916a8c65ef3653ce4911d8f85a520ab39ee6cdb","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-h75t75dg5lvd/cluster-candidate-4-22-bc8pwv should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00155ce80\u003e: \n failed to create HCP cluster cluster-candidate-4-22-bc8pwv: failed starting cluster creation \"cluster-candidate-4-22-bc8pwv\" in resourcegroup=\"rg-candidate-4-22-h75t75dg5lvd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-h75t75dg5lvd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-22-bc8pwv\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-h75t75dg5lvd/cluster-candidate-4-22-bc8pwv should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00155ce80\u003e: \n failed to create HCP cluster cluster-candidate-4-22-bc8pwv: failed starting cluster creation \"cluster-candidate-4-22-bc8pwv\" in resourcegroup=\"rg-candidate-4-22-h75t75dg5lvd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-h75t75dg5lvd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-22-bc8pwv\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"1cab1a94054d21fd85144b563dd422dcb764b4c59c039d42cc7986bdc604e6ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"2b9093bf5c45c0ab4c53608afd8efb00f9e181eec19f65cc03ce4c6f28da0828","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-5hgv94\" with version 4.22.13 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c6a200\u003e: \n failed to create HCP cluster cluster-zstream-4-22-5hgv94: failed starting cluster creation \"cluster-zstream-4-22-5hgv94\" in resourcegroup=\"rg-zstream-upgrade-4-22-rg8drsghclvc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-rg8drsghclvc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-5hgv94\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-5hgv94\" with version 4.22.13 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c6a200\u003e: \n failed to create HCP cluster cluster-zstream-4-22-5hgv94: failed starting cluster creation \"cluster-zstream-4-22-5hgv94\" in resourcegroup=\"rg-zstream-upgrade-4-22-rg8drsghclvc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-rg8drsghclvc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-5hgv94\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"1d7a01892091714b3d20a3373c140c5ab9aa03c73b3eb556b9b6b2f689502323","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"cd1c7583f2647a6092127bb87aa94ca9928e0cb7d3b98f709bb1673f365c5faf","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0002a67a0\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed starting cluster creation \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-jdv6zjq9vb9z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-jdv6zjq9vb9z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0002a67a0\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed starting cluster creation \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-jdv6zjq9vb9z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-jdv6zjq9vb9z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"221992559ec73f25e7e1dab8e14623e4e6db214f46fa71c88dfafc6300ddcc36","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"21ab44da72b7e9ae397267d0f0a8a18bded933ed8452908aebd7aabb096785c7","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ba2aa0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed starting cluster creation \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-mgkv5vcnbk4t\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-mgkv5vcnbk4t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-delete-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ba2aa0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed starting cluster creation \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-mgkv5vcnbk4t\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-mgkv5vcnbk4t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-delete-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"2a271d58b7df3dda0bfe8b4722912d25ad0fe865bd40d3750dacacfff48300fd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"eea94e83586857a6e7bd796c43608d6810be52d625ea8b1c7bc0e154a36458eb","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b89040\u003e: \n failed to create HCP cluster ea-listnksgg8: failed starting cluster creation \"ea-listnksgg8\" in resourcegroup=\"ea-list-tx57g2bcffrp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-tx57g2bcffrp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listnksgg8\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b89040\u003e: \n failed to create HCP cluster ea-listnksgg8: failed starting cluster creation \"ea-listnksgg8\" in resourcegroup=\"ea-list-tx57g2bcffrp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-tx57g2bcffrp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listnksgg8\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"2bf6a1c935fe62aab1c01143e281daf77382bd50eb87bd3552f5b55d3da9e9ff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"baeffd426c735122a05b9160488c4dbd60bc4728a5b3d988d4e82aba7d0786a4","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-zd2hv5 at version 4.22.14\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001018dc0\u003e: \n failed to create HCP cluster np-skip-minor-zd2hv5: failed starting cluster creation \"np-skip-minor-zd2hv5\" in resourcegroup=\"rg-np-skip-minor-zd2hv5-xg6gmd9sxw8t\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-zd2hv5-xg6gmd9sxw8t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-zd2hv5\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-zd2hv5 at version 4.22.14\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001018dc0\u003e: \n failed to create HCP cluster np-skip-minor-zd2hv5: failed starting cluster creation \"np-skip-minor-zd2hv5\" in resourcegroup=\"rg-np-skip-minor-zd2hv5-xg6gmd9sxw8t\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-zd2hv5-xg6gmd9sxw8t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-zd2hv5\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"2ca112bc99bf8e1a1086c565054efa8af8528da7613e1be7f2ab39aaa0262089","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"4fa3bb0bbd8451703ed73fb305781833110a4c051383b3123925afe42add42fd","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f0e200\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-2w6r47m2vzbz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-2w6r47m2vzbz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f0e200\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-2w6r47m2vzbz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-2w6r47m2vzbz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"2ffdf8dec0642402f4d0b61ee93d8b028f22a99386b766c69b53474f5ce09f61","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"cffe34c5297814524ceaaab2392bfcacbf1ba1c268e4ea9471d44f96f9f151cf","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000640360\u003e: \n failed to create HCP cluster np-downgrade-k7lqv2: failed starting cluster creation \"np-downgrade-k7lqv2\" in resourcegroup=\"rg-np-downgrade-k7lqv2-wq4hwbkxc846\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-k7lqv2-wq4hwbkxc846/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-k7lqv2\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000640360\u003e: \n failed to create HCP cluster np-downgrade-k7lqv2: failed starting cluster creation \"np-downgrade-k7lqv2\" in resourcegroup=\"rg-np-downgrade-k7lqv2-wq4hwbkxc846\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-k7lqv2-wq4hwbkxc846/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-k7lqv2\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"33e49622d440c950f9ac3d9f85946775865e2e2a3e7a4cbcdf096ea64b19940e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"15919028e1ac14fcd30d83d00063e34c3905a3b42a1fe2c678678bef887c5398","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000707e00\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-pg4bl4582c6x\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-pg4bl4582c6x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000707e00\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-pg4bl4582c6x\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-pg4bl4582c6x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"34e460e1a24470fbe64027e175a7655522272a01a7293ad5e0af9d4c4e6ebbf1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"67553b55a2af77864f0735e878cf7f4a32d0f0ca673ead8a06f3d92d8594e2da","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-6ltghn4pvj58/cluster-candidate-4-21-s5gc4g should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00072aea0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-s5gc4g: failed starting cluster creation \"cluster-candidate-4-21-s5gc4g\" in resourcegroup=\"rg-candidate-4-21-6ltghn4pvj58\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-6ltghn4pvj58/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-21-s5gc4g\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-6ltghn4pvj58/cluster-candidate-4-21-s5gc4g should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00072aea0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-s5gc4g: failed starting cluster creation \"cluster-candidate-4-21-s5gc4g\" in resourcegroup=\"rg-candidate-4-21-6ltghn4pvj58\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-6ltghn4pvj58/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-21-s5gc4g\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"38ed3b7d1c83d09928686aa4099f8d167385fb9f4a6c8c7b0168a630897f95bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"2d70c4931be129673294f92930d75ded19384939b39f6fcc68c306fe7a9a8917","test_name":"Customer should be able to update node pool labels and taints","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eacda0\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed starting cluster creation \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-4wwnsq5nbvwk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-4wwnsq5nbvwk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eacda0\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed starting cluster creation \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-4wwnsq5nbvwk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-4wwnsq5nbvwk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"3988084b10b05672cb90e35e81369afd912677fb47f78e9c4ac6f8bb8d1752d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"71525a5147dd46a524bfebfb7a7e9ac475ab1cd78c5c49603707efaaac058811","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c242a0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed starting cluster creation \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-jh5p965r9h5p\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-jh5p965r9h5p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c242a0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed starting cluster creation \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-jh5p965r9h5p\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-jh5p965r9h5p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"478f0d7d2a6af03c4fb5ecc765e6bc89aa912619f2d51d9dded7cdcf6e75e91f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"7ded978aba44423367e409626a9a3315d57d345e256488391ce34265653bbcd7","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009f61e0\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed starting cluster creation \"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-plvwdtqxskcm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-plvwdtqxskcm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009f61e0\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed starting cluster creation \"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-plvwdtqxskcm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-plvwdtqxskcm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"5456c07e3e2329382527631e8104a7f293be24ef2dedaffa5bc44f96c75c8571","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"fa31b30e1f0f06a0d18a6453b7dd232180a117910392ffe556124807a57f5051","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-hrgb26 with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f8a340\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-hrgb26: failed starting cluster creation \"np-version-upgrade-cluster-hrgb26\" in resourcegroup=\"rg-np-version-upgrade-hrgb26-88r5lslmqpf7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-hrgb26-88r5lslmqpf7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-hrgb26\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-hrgb26 with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f8a340\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-hrgb26: failed starting cluster creation \"np-version-upgrade-cluster-hrgb26\" in resourcegroup=\"rg-np-version-upgrade-hrgb26-88r5lslmqpf7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-hrgb26-88r5lslmqpf7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-hrgb26\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"54dfa3e55a1497a675c03d3913808534e2055edab495ca6518c4af5dcc380e54","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"03b77825464cd5d0f0c913cdaae7947a72c014fdb9444903e3712d0c0fe522ed","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f101e0\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed starting cluster creation \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-8m2tcg7xkh2f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-8m2tcg7xkh2f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cx-rg-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f101e0\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed starting cluster creation \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-8m2tcg7xkh2f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-8m2tcg7xkh2f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cx-rg-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6099adb5a9109ffc4ff570f51d650d38935fd6c9b89163474e9ec3c48464889a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"11dd84afe0645df3b655944fb22194c0f2e58c9268e44c457850cea2b7c34e78","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d920c0\u003e: \n failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-dh726mjb6hfc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-dh726mjb6hfc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d920c0\u003e: \n failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-dh726mjb6hfc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-dh726mjb6hfc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"67f1de21eb6f25b93715690d278f30077b5c0fc4a823b67523d4055ffc48f98d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"216725343df0529e9b36f0e7e1e13131bf0ce5434bc431bcba8f418e8e64c19a","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00083e4e0\u003e: \n failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-mqdcg88p65p5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-mqdcg88p65p5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00083e4e0\u003e: \n failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-mqdcg88p65p5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-mqdcg88p65p5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6b5944f209711026beaf10f4e83e3cf3489371d3870c57b2eb171530732d0d85","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"31f4483198750e6aaf7406ab17a19a3d028274697f1c29c9b1c89a59d30f4bed","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000da21c0\u003e: \n failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \"patch-tags-cluster\" in resourcegroup=\"patch-tags-wzckxjx8ln9d\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-wzckxjx8ln9d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000da21c0\u003e: \n failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \"patch-tags-cluster\" in resourcegroup=\"patch-tags-wzckxjx8ln9d\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-wzckxjx8ln9d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"750798b146238ac36bff20eabafb79149b6df0ef789e2645f03f01359a1dcd9d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"8046cfb505c99b60ca80e943e2bf1314cc6c403f4aa343db74d259ea4a6e93f3","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-plbg5s\" with version 4.20.37 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014c45e0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-plbg5s: failed starting cluster creation \"cluster-zstream-4-20-plbg5s\" in resourcegroup=\"rg-zstream-upgrade-4-20-5dctr2v29hxm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-5dctr2v29hxm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-plbg5s\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-plbg5s\" with version 4.20.37 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014c45e0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-plbg5s: failed starting cluster creation \"cluster-zstream-4-20-plbg5s\" in resourcegroup=\"rg-zstream-upgrade-4-20-5dctr2v29hxm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-5dctr2v29hxm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-plbg5s\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7e2276012f64a175ed26b48e04a71c02f29e618551818dcbc45a85e7aa267fa5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"35acdd5271b6215a49c6cf08b71aced0f1d2578371ad1ceae5b07fceff8cccef","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000726720\u003e: \n failed starting cluster creation \"agg-cluster\" in resourcegroup=\"feature-aggregation-6mvbgcknfnql\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-6mvbgcknfnql/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000726720\u003e: \n failed starting cluster creation \"agg-cluster\" in resourcegroup=\"feature-aggregation-6mvbgcknfnql\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-6mvbgcknfnql/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"8122a12d1c0831a005eac942f27f64dd89fd11fb9479d5e20e5e6dcdb1f59d2a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"92f59da349e0622f853a8c3a1bec0a0e8c4348115990bbdcc3187438162d5e69","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010ca2a0\u003e: \n failed starting cluster creation \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-wjdnbdcz7v5v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-wjdnbdcz7v5v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-20\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010ca2a0\u003e: \n failed starting cluster creation \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-wjdnbdcz7v5v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-wjdnbdcz7v5v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-20\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"888757659e64169659088d1f39a2d3e29433fdfc1a631d416433ff1e94cb8d81","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"fd2a8703e1cfa9007dc900c3faf403c43e94c2f939e63eb0a30b4489f161a1ce","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f341e0\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed starting cluster creation \"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-k4jvltkd7tqn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-k4jvltkd7tqn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/missing-mi-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f341e0\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed starting cluster creation \"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-k4jvltkd7tqn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-k4jvltkd7tqn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/missing-mi-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"88db1e0c1f54f4a81d4fa56d85b1bd9cc39166b9da6fc8792c9b0b774f40fae1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"9c08634571d13f7a120ae29ee62838a0103a629a513f3f84e21651285d4b288f","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006f8420\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-chbpbwgpb8pr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-chbpbwgpb8pr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006f8420\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-chbpbwgpb8pr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-chbpbwgpb8pr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"915bc8119f612c0d47375e00a9806ed8cce7c0cdda1375bc5c4723f752d9313c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"9094d9c9d14a7b83f4e3a161a729b7277c10c8d146f7017e283f7a8344d42a64","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-xvzwmw with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f141a0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-xvzwmw: failed starting cluster creation \"np-version-upgrade-cluster-xvzwmw\" in resourcegroup=\"rg-np-version-upgrade-xvzwmw-bxjhmmlcjtxn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-xvzwmw-bxjhmmlcjtxn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-xvzwmw\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-xvzwmw with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f141a0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-xvzwmw: failed starting cluster creation \"np-version-upgrade-cluster-xvzwmw\" in resourcegroup=\"rg-np-version-upgrade-xvzwmw-bxjhmmlcjtxn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-xvzwmw-bxjhmmlcjtxn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-xvzwmw\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"9178eedca9603e2df240aad5c5b48d0e4bddd033c711c35e3e974dec9da135eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"d3ca56e7987ce4db58ef5dd7aadd3fd270620b1749223ce2e302a818b775a91a","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b48360\u003e: \n failed starting cluster creation \"private-kv-cluster\" in resourcegroup=\"private-keyvault-84h94kdhhq2z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-84h94kdhhq2z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b48360\u003e: \n failed starting cluster creation \"private-kv-cluster\" in resourcegroup=\"private-keyvault-84h94kdhhq2z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-84h94kdhhq2z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"93e0ce83410805235258b6b49ad573ca90b9f3e272342ae5e46c33a9e88ff862","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"0cfac70bd1340931647bfba310152659cb81acd7ef73020d9fec20384f21a480","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001000160\u003e: \n failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-lbjqxnvvbh9l\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-lbjqxnvvbh9l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001000160\u003e: \n failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-lbjqxnvvbh9l\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-lbjqxnvvbh9l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"95cc10098d43ec8ff4e207157b1debdaaf7d7b9d8ef4518844d2ad43fdcfb7ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"324c00bc6b17f4c27a79bfb45836691b50ecbc4134a78456ae7c1b0d269b1e69","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d26300\u003e: \n failed to create HCP cluster cilium-cl: failed starting cluster creation \"cilium-cl\" in resourcegroup=\"cni-cilium-kxlbk5nhx7dg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-kxlbk5nhx7dg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cl\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d26300\u003e: \n failed to create HCP cluster cilium-cl: failed starting cluster creation \"cilium-cl\" in resourcegroup=\"cni-cilium-kxlbk5nhx7dg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-kxlbk5nhx7dg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cl\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"9956f8d8aa9f39e4930400dca896c7d40740085815ceed7be859d94c323c981f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"8dfb46dbacd72001834d323a20b869a0f996359144e2afd458b2448b4dc98cf2","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aaa160\u003e: \n failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-ww5k9bfrs47z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-ww5k9bfrs47z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aaa160\u003e: \n failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-ww5k9bfrs47z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-ww5k9bfrs47z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"9a9691218278ff5d97f8625a64da95f18786e19fbe4b1589d389ba6b14f50a82","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"bfefd8b73a95855d933ee982b50b0f0e5e07d0da48727c260ed1c8f8648facfd","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002880260\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-98wggrdxqbzz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-98wggrdxqbzz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002880260\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-98wggrdxqbzz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-98wggrdxqbzz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"9c2dee0b44fe61cde55e110f5473113431e4c4b43dae599139abea52e0b8f041","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"c09fb79396c6290ed844ec65b79dc38591d982d0e590dc59031e5715e882b2cf","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00103c2a0\u003e: \n failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-lhpbhxvtfvqm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-lhpbhxvtfvqm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00103c2a0\u003e: \n failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-lhpbhxvtfvqm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-lhpbhxvtfvqm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"9dcc5a3b115d86d38effb58d9e37b923c81b8e97abc2d2e6dbc178834774828c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"795c08d8819e681ed7485b9c8f3dd820bc16a481608400dbde3cb44146d78088","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-j8stln\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001092140\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-j8stln: failed starting cluster creation \"cp-ystream-upgrade-4-21-j8stln\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-k6fzpk2b68m4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-k6fzpk2b68m4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-j8stln\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-j8stln\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001092140\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-j8stln: failed starting cluster creation \"cp-ystream-upgrade-4-21-j8stln\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-k6fzpk2b68m4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-k6fzpk2b68m4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-j8stln\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"a2f810c0f2859b93e533fb896875cbe37b66f88cf6d4fecb16bd50b8ae644e46","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"a31a984f939916a0796de23b382a377251226044e65fbd176755803e62ab7118","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00118a260\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-q464fsngnb2z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-q464fsngnb2z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00118a260\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-q464fsngnb2z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-q464fsngnb2z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"a4c1cd7226f2576a645fe03a45000ab4264aeeee7980e7d80f1817cf2e6fa201","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"afe89bb70726178b639cae42abd86fb26a45875403133fee89c17bb3f0dabdc1","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-mckpkv\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000860720\u003e: \n failed to create HCP cluster np-noedge-mckpkv: failed starting cluster creation \"np-noedge-mckpkv\" in resourcegroup=\"rg-np-noedge-mckpkv-chk6hnblbkth\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-mckpkv-chk6hnblbkth/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-mckpkv\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-mckpkv\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000860720\u003e: \n failed to create HCP cluster np-noedge-mckpkv: failed starting cluster creation \"np-noedge-mckpkv\" in resourcegroup=\"rg-np-noedge-mckpkv-chk6hnblbkth\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-mckpkv-chk6hnblbkth/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-mckpkv\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"af28071603256f05361ce71bef02f8b9002fc60748a97f2cf1f0bd705bfcd74b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"ff22fcf5e35af3fbcc3cebd0bd3dd0c5eb072b99885e355b90e687d8b5d3d950","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0010303c0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-6t49grdzfqmk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0010303c0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-6t49grdzfqmk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"bb9e45daa2e0999da6f9538dfd6add53360f88dd5bf2e7958f381bbd43760eb8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"ec3a67d29e9d134e429f91cd72b99e2438ffd5f649f95915d71f08fe6389f0a6","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001012360\u003e: \n failed to create HCP cluster ea-cluster: failed starting cluster creation \"ea-cluster\" in resourcegroup=\"external-auth-cluster-d5hfbw5gmtt5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-d5hfbw5gmtt5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001012360\u003e: \n failed to create HCP cluster ea-cluster: failed starting cluster creation \"ea-cluster\" in resourcegroup=\"external-auth-cluster-d5hfbw5gmtt5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-d5hfbw5gmtt5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c4d89e649ae21f00b1f6033e73e5df620c4bcc27cdb680f9d39fcd82f6f9ef2d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"d6230e5e7e710e0d42bff7a6002f4e38a4b5e9da3efde2e66bce2f28fcaa7db9","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00062c5e0\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-lczl6v6nrz6w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-lczl6v6nrz6w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00062c5e0\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-lczl6v6nrz6w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-lczl6v6nrz6w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ca6869f262ca85c8954303f7280cbe1bed0f6e85b57061cf046ea00756dab15d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"ef22972e47abc2792176e133a01c85a9b2384bd770dce3cada367ce224007462","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00092c680\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-czpph9wzgls4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-czpph9wzgls4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00092c680\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-czpph9wzgls4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-czpph9wzgls4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"cdf14cbb7f7f5b07379a838592cd4ad5ca9ecd10ed0816d1286c74431d627a4c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"ec99d240f7e03e54f55c1ce1e3f244fb13a98e2e3e13f3340e7c22ecadfa7ec9","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001757c20\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001757c20\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ce6f3c6b6fd5d9d7fedb28b73c386845bb0480b3847b3f07720d025141c33fdc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"b4e7af59ddeae0ea6bc250fac9809dc26653c15bcc599bcdf62dffbe67f2017a","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b1af80\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-v2bhlsm92ksh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-v2bhlsm92ksh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b1af80\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-v2bhlsm92ksh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-v2bhlsm92ksh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d2415c8ceba9a86ca09ddb4abdd0967952005ef911f2be632c320db178222cab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"1662695320a9d015fc05041d761e86397e300ef2d8aa4efd29dbe9589d289ba0","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-cxl549\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011c8240\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-cxl549: failed starting cluster creation \"cp-ystream-upgrade-5-0-cxl549\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-klrb5nkdm6fv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-klrb5nkdm6fv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-cxl549\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-cxl549\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011c8240\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-cxl549: failed starting cluster creation \"cp-ystream-upgrade-5-0-cxl549\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-klrb5nkdm6fv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-klrb5nkdm6fv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-cxl549\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d52966cb7faba977e98725c6fd545699032a06e02e0c6c00715ba2afc6f13f2e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"5fb1c2770708686f8963accd704a5ec1da9ef9cc1f1cd732cce61f719fb651a6","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-c2r948\" with version 4.21.33 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ed43c0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-c2r948: failed starting cluster creation \"cluster-zstream-4-21-c2r948\" in resourcegroup=\"rg-zstream-upgrade-4-21-swxb5fxkxx4z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-swxb5fxkxx4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-c2r948\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-c2r948\" with version 4.21.33 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ed43c0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-c2r948: failed starting cluster creation \"cluster-zstream-4-21-c2r948\" in resourcegroup=\"rg-zstream-upgrade-4-21-swxb5fxkxx4z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-swxb5fxkxx4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-c2r948\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d7a6b930bead2ae1438c206cb821cbf32dd2ac72176a3f9503b2cf847ec64760","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"0ab4166b7eaa370dfdb49ad732a32dad0ac91724444ddf819aa3c29f3a9f6a70","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fde240\u003e: \n failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-qj86j7vf296r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-qj86j7vf296r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fde240\u003e: \n failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-qj86j7vf296r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-qj86j7vf296r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"db8575be0e9a42191cdaedbb734f3a51a7974cf76813d2dac2962623ccdc1c96","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"483d93133052646e81580234624852295a4a3911c14ae9d5bfcd4ed0a65bbd82","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-6bkkt2 with version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013940e0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-6bkkt2: failed starting cluster creation \"np-version-upgrade-cluster-6bkkt2\" in resourcegroup=\"rg-np-version-upgrade-6bkkt2-m96mr2dxx5b8\": Put \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-6bkkt2-m96mr2dxx5b8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-6bkkt2?api-version=2024-06-10-preview\": stream error: stream ID 37; CANCEL; received from peer\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-6bkkt2 with version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013940e0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-6bkkt2: failed starting cluster creation \"np-version-upgrade-cluster-6bkkt2\" in resourcegroup=\"rg-np-version-upgrade-6bkkt2-m96mr2dxx5b8\": Put \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-6bkkt2-m96mr2dxx5b8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-6bkkt2?api-version=2024-06-10-preview\": stream error: stream ID 37; CANCEL; received from peer\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8d719d468a5fb10bc89ca62a043bd8700c617674d20a67919750ac1d3434857f","failure_pattern":"failed to create HCP cluster \u003ccluster\u003e: failed starting cluster creation \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\": Put \"\u003curl\u003e\": stream error: stream ID 37; CANCEL; received from peer","search_query":"failed to create HCP cluster np-version-upgrade-cluster-6bkkt2: failed starting cluster creation \"np-version-upgrade-cluster-6bkkt2\" in resourcegroup=\"rg-np-version-upgrade-6bkkt2-m96mr2dxx5b8\": Put \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-6bkkt2-m96mr2dxx5b8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-6bkkt2?api-version=2024-06-10-preview\": stream error: stream ID 37; CANCEL; received from peer"}},{"row_id":"dc057d0067ea80d793c03ac022d77fa77b1f75e7fc2ac85c84bbbd2c71a4bb07","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"cf9272d14bae7f20121b2149ee98cf928951ebf46d29b57c025421aa4ae43d1e","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00113a320\u003e: \n failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-kbl8bmnn4b4w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-kbl8bmnn4b4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00113a320\u003e: \n failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-kbl8bmnn4b4w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-kbl8bmnn4b4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"dc67a0385ec0277679e028629e37600dbf8c850a1b831c4fce7f405772b85c50","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"e97a9ed671fb6a24f0766a2b032cfb0e6df40f68c200b713382036952f0bfc30","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-g28hnvmc7v6c/cluster-candidate-4-20-762dl6 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ca4160\u003e: \n failed to create HCP cluster cluster-candidate-4-20-762dl6: failed starting cluster creation \"cluster-candidate-4-20-762dl6\" in resourcegroup=\"rg-candidate-4-20-g28hnvmc7v6c\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-g28hnvmc7v6c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-20-762dl6\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-g28hnvmc7v6c/cluster-candidate-4-20-762dl6 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ca4160\u003e: \n failed to create HCP cluster cluster-candidate-4-20-762dl6: failed starting cluster creation \"cluster-candidate-4-20-762dl6\" in resourcegroup=\"rg-candidate-4-20-g28hnvmc7v6c\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-g28hnvmc7v6c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-20-762dl6\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"e3d0c64ae6475595d5466705cd3fb67145b3aeb604b7f965c861e7b222f829bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"a872f17e452fb889fc8ba0a1d6f796a5739c45d0dbddeda106b88f5f9d119623","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000da2400\u003e: \n failed to create HCP cluster np-dg-minor-qh8fbr: failed starting cluster creation \"np-dg-minor-qh8fbr\" in resourcegroup=\"rg-np-dg-minor-qh8fbr-m5zhfcgf52j7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-qh8fbr-m5zhfcgf52j7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-qh8fbr\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000da2400\u003e: \n failed to create HCP cluster np-dg-minor-qh8fbr: failed starting cluster creation \"np-dg-minor-qh8fbr\" in resourcegroup=\"rg-np-dg-minor-qh8fbr-m5zhfcgf52j7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-qh8fbr-m5zhfcgf52j7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-qh8fbr\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"eafead63afb6a5af37f79d83b0f3f7cde0a190b7674bd0bb4986210e85287216","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"6ba691bb4d98d95c22f5be1e9f862a0628528e2a58e45d0cc27810b53fc1964f","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-h79k8k\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014d33a0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-h79k8k: failed starting cluster creation \"cp-ystream-upgrade-4-22-h79k8k\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-w48jfg4cknrl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-w48jfg4cknrl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-h79k8k\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-h79k8k\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014d33a0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-h79k8k: failed starting cluster creation \"cp-ystream-upgrade-4-22-h79k8k\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-w48jfg4cknrl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-w48jfg4cknrl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-h79k8k\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ed2dfc75f825f4107c66cee2895210ac41ebb4000e6f777f9cc42b22cad640c1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"1438e991fc10aac23e7722574797bd29edc9098daf2172e0fd8c3ae2085d262c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-pxrfzhr4lgdl/cluster-candidate-5-0-gsk4z7 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00097ccc0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-gsk4z7: failed starting cluster creation \"cluster-candidate-5-0-gsk4z7\" in resourcegroup=\"rg-candidate-5-0-pxrfzhr4lgdl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-pxrfzhr4lgdl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-5-0-gsk4z7\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-pxrfzhr4lgdl/cluster-candidate-5-0-gsk4z7 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00097ccc0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-gsk4z7: failed starting cluster creation \"cluster-candidate-5-0-gsk4z7\" in resourcegroup=\"rg-candidate-5-0-pxrfzhr4lgdl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-pxrfzhr4lgdl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-5-0-gsk4z7\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f6edc55771699cbee351520efa2dde2386f80de928e2cff4671d8f0c3ea1769c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"9cf81b6880288565cab800db518d46590a69e97af1a7c21f4d2571d4e91ad36c","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:105]: failed to begin creating HCP cluster \"admin-cred-lifecycle-q52tkz\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001098400\u003e: \n failed starting cluster creation \"admin-cred-lifecycle-q52tkz\" in resourcegroup=\"admin-credential-lifecycle-test-ccbpf2vrpk7m\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-ccbpf2vrpk7m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-q52tkz\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:105]: failed to begin creating HCP cluster \"admin-cred-lifecycle-q52tkz\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001098400\u003e: \n failed starting cluster creation \"admin-cred-lifecycle-q52tkz\" in resourcegroup=\"admin-credential-lifecycle-test-ccbpf2vrpk7m\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-ccbpf2vrpk7m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-q52tkz\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"fcddb98298c65272a9c0bfecb8b2966aac50ec5cc2c96fef0c6427cee050106a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"16927c15887b574382c340bf94b042f4c251206eb455c46dbf767b8bacdd00e7","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a6e540\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-lq7mxkdms4nx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-lq7mxkdms4nx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a6e540\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-lq7mxkdms4nx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-lq7mxkdms4nx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"fd071897c554ee592e5a62b9f69736d2f08a2223821ae011e5d615480b863127","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","failed_at":"e2e","signature_id":"c00047326594e77e168a68c8731720f644c8614a7bc5587fe8aac8ff5098493d","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001956280\u003e: \n failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-xfb8qbsjtxct\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-xfb8qbsjtxct/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001956280\u003e: \n failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-xfb8qbsjtxct\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-xfb8qbsjtxct/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}}],"failure_pattern_summary":{"signature_count":56,"failure_pattern_ids":["82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","8d719d468a5fb10bc89ca62a043bd8700c617674d20a67919750ac1d3434857f"],"matched_occurrences":56,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}