{"meta":{"date":"2026-09-16","timezone":"UTC","generated_at":"2026-10-15T02:56:40Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":144,"failed_runs":120,"runs_with_occurrences":120,"runs_with_failure_pattern_matches":102,"runs_with_unmatched_occurrences":18,"runs_without_occurrence_rows":24,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100373409937494016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6977,"pr_state":"closed","pr_sha":"557c04379e033b9de0bd85904b49103f2cdef1b1","final_merged_sha":"557c04379e033b9de0bd85904b49103f2cdef1b1","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T23:56:58Z","started_at":"2026-09-16T23:56:58Z","completed_at":"2026-09-17T02:55:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:06.604117121Z","timing_metadata_checked_at":"2026-09-18T15:41:06.604117121Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:06.608247136Z","region_metadata_checked_at":"2026-09-18T15:41:06.608247136Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"d0d2c6902be861e19fed212e6a37d2fab28d8d3ab88e851282d72796affb1621","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100373409937494016","occurred_at":"2026-09-16T23:56:58Z","failed_at":"alert","signature_id":"1e3921ef26cf400f3e3eaae679c2c7938f1ecf3228c5ec7fae0e28ef8fa1159f","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 12 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T02:14:20Z\n Ended: 2026-09-17T02:34:22Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7494016-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-kv8ld\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-kv8ld (namespace arobit) on cluster ci01-j7494016-svc has averaged 177.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-17T02:14:20Z\n Ended: 2026-09-17T02:32:20Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7494016-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-7b8f98bb8d-xxtkh\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-7b8f98bb8d-xxtkh (namespace mgmt-agent) on cluster ci01-j7494016-mgmt-1 has averaged 189.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-17T02:14:20Z\n Ended: 2026-09-17T02:40:19Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7494016-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-8987ccb77-nmgwd\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-8987ccb77-nmgwd (namespace clusters-service) on cluster ci01-j7494016-svc has averaged 256.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-17T02:14:20Z\n Ended: 2026-09-17T02:29:21Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7494016-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-7b8f98bb8d-l5m85\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-7b8f98bb8d-l5m85 (namespace mgmt-agent) on cluster ci01-j7494016-mgmt-2 has averaged 180% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-17T02:14:20Z\n Ended: 2026-09-17T02:33:21Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7494016-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-hmnhn\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-hmnhn (namespace arobit) on cluster ci01-j7494016-mgmt-1 has averaged 217.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Resolved\n Started: 2026-09-17T02:14:20Z\n Ended: 2026-09-17T02:29:21Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7494016-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-d9thl\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-d9thl (namespace arobit) on cluster ci01-j7494016-mgmt-2 has averaged 167.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-09-17T02:14:20Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7494016-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-85694f5cf9-96x49\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-85694f5cf9-96x49 (namespace aro-hcp) on cluster ci01-j7494016-svc has averaged 418.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 8:\n State: Resolved\n Started: 2026-09-17T02:14:20Z\n Ended: 2026-09-17T02:40:19Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7494016-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-8987ccb77-btgnn\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-8987ccb77-btgnn (namespace clusters-service) on cluster ci01-j7494016-svc has averaged 253.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 9:\n State: Resolved\n Started: 2026-09-17T02:14:20Z\n Ended: 2026-09-17T02:30:19Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7494016-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-w55jk\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-w55jk (namespace arobit) on cluster ci01-j7494016-mgmt-1 has averaged 147.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 10:\n State: Resolved\n Started: 2026-09-17T02:14:20Z\n Ended: 2026-09-17T02:27:20Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7494016-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-bwg62\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-bwg62 (namespace arobit) on cluster ci01-j7494016-svc has averaged 130.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 11:\n State: Fired (not resolved)\n Started: 2026-09-17T02:14:20Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7494016-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-8987ccb77-sfcv9\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-8987ccb77-sfcv9 (namespace clusters-service) on cluster ci01-j7494016-svc has averaged 850.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 12:\n State: Resolved\n Started: 2026-09-17T02:14:20Z\n Ended: 2026-09-17T02:33:21Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7494016-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-v5ccx\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-v5ccx (namespace arobit) on cluster ci01-j7494016-mgmt-1 has averaged 160.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100358368177688576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6964,"pr_state":"closed","pr_sha":"af707ce72df7d034eaee96beafbefdfe37fc0800","final_merged_sha":"af707ce72df7d034eaee96beafbefdfe37fc0800","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T22:57:12Z","started_at":"2026-09-16T22:57:12Z","completed_at":"2026-09-17T00:02:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:58.93432461Z","timing_metadata_checked_at":"2026-09-18T15:40:58.93432461Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:58.937794923Z","region_metadata_checked_at":"2026-09-19T03:30:38.48177921Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"419c146aa4c23310c2fe794e1f687a6ce88ab02215928f3eceeebded447db022","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100358368177688576","occurred_at":"2026-09-16T22:57:12Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100358303195336704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6966,"pr_state":"closed","pr_sha":"c93ff943c35ad2d21e3a98df5a27db691c9749a7","final_merged_sha":"c93ff943c35ad2d21e3a98df5a27db691c9749a7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T22:56:56Z","started_at":"2026-09-16T22:56:56Z","completed_at":"2026-09-17T00:03:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:00.562708967Z","timing_metadata_checked_at":"2026-09-18T15:41:00.562708967Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:00.565928279Z","region_metadata_checked_at":"2026-09-19T03:30:38.841691237Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"7039d24657960b9f0d2ffa8794fa31ba1c3020dc6f7b23c62f6c8dfbb320c0c7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100358303195336704","occurred_at":"2026-09-16T22:56:56Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100358299223330816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6976,"pr_state":"open","pr_sha":"3f37770fba22a55b4336e3a85cbf275bf37c8279","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T22:56:55Z","started_at":"2026-09-16T22:56:55Z","completed_at":"2026-09-17T00:00:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:05.199618964Z","timing_metadata_checked_at":"2026-09-18T15:41:05.199618964Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:05.202861675Z","region_metadata_checked_at":"2026-09-19T03:30:39.661153744Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"60bce27c2e6256fd451d600c45ab56f0ad3581ea1264b571467e9fe2f9fa839c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100358299223330816","occurred_at":"2026-09-16T22:56:55Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100358273944260608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6933,"pr_state":"closed","pr_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","final_merged_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T22:56:49Z","started_at":"2026-09-16T22:56:49Z","completed_at":"2026-09-17T00:01:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:43.700937218Z","timing_metadata_checked_at":"2026-09-18T15:40:43.700937218Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:43.704548431Z","region_metadata_checked_at":"2026-09-19T03:30:36.929832187Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"f2a7cd9d03c36a1379a26b18f84f37b7cfe3d6d07cca71711a3a802de39c7c5b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100358273944260608","occurred_at":"2026-09-16T22:56:49Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100358247872466944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6791,"pr_state":"closed","pr_sha":"764ee60fa32d166f92c6c286f96007787ee189de","final_merged_sha":"00c5d478babb26a92f08e45ee9f4c322bafd7507","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T22:56:43Z","started_at":"2026-09-16T22:56:43Z","completed_at":"2026-09-17T00:01:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:23.186203411Z","timing_metadata_checked_at":"2026-09-18T15:40:23.186203411Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:23.189297022Z","region_metadata_checked_at":"2026-09-19T03:30:34.534165053Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"87deac8d134d517e1021a6e772297f11e62a7904aa6fca4d7104fef0540a60e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100358247872466944","occurred_at":"2026-09-16T22:56:43Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100358238510780416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6980,"pr_state":"closed","pr_sha":"4c6ef86c4410d1e48dd7653c177a4370cf38a246","final_merged_sha":"4c6ef86c4410d1e48dd7653c177a4370cf38a246","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T22:56:41Z","started_at":"2026-09-16T22:56:41Z","completed_at":"2026-09-17T00:01:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:07.902032495Z","timing_metadata_checked_at":"2026-09-18T15:41:07.902032495Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:07.907494115Z","region_metadata_checked_at":"2026-09-19T03:30:39.997742377Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"3500eddb67725d39ab9f86db9c2e09d19c1cd9e9c7b7e773119fbc614a563bc0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100358238510780416","occurred_at":"2026-09-16T22:56:41Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100358225168699392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6752,"pr_state":"closed","pr_sha":"d433c21668cfdc66f98262f9099c8086b4397248","final_merged_sha":"d433c21668cfdc66f98262f9099c8086b4397248","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T22:56:38Z","started_at":"2026-09-16T22:56:38Z","completed_at":"2026-09-17T00:02:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:20.375400798Z","timing_metadata_checked_at":"2026-09-18T15:40:20.375400798Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:20.37882251Z","region_metadata_checked_at":"2026-09-19T03:30:34.183168458Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"4ecddc41c94a45a568906be097d016c44f3e35f72c8140dd8fe664193b001512","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100358225168699392","occurred_at":"2026-09-16T22:56:38Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100357907810881536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6939,"pr_state":"closed","pr_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","final_merged_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T22:55:22Z","started_at":"2026-09-16T22:55:22Z","completed_at":"2026-09-17T00:00:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:44.533784915Z","timing_metadata_checked_at":"2026-09-18T15:40:44.533784915Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:44.537069427Z","region_metadata_checked_at":"2026-09-19T03:30:37.417076084Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"4ff13430db3180a739019405b7e5adcb269ecf32170f5d815b137809003e2134","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100357907810881536","occurred_at":"2026-09-16T22:55:22Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100354655547559936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6940,"pr_state":"closed","pr_sha":"62cdd448d25099f3b77adb5f5e21439e857c6d5f","final_merged_sha":"62cdd448d25099f3b77adb5f5e21439e857c6d5f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T22:42:23Z","started_at":"2026-09-16T22:42:23Z","completed_at":"2026-09-17T00:44:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:46.312526613Z","timing_metadata_checked_at":"2026-09-18T15:40:46.312526613Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:46.316194826Z","region_metadata_checked_at":"2026-09-18T15:40:46.316194826Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"aebc5c2821cadfd50b6ee4373e6fd06153764a355b03b59594e1185fb1af344c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100354655547559936","occurred_at":"2026-09-16T22:42:23Z","failed_at":"alert","signature_id":"b66349955715ed443f1362f9a218926c741bbba15a7053b8231a572f7bce0c04","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T23:46:37Z\n Ended: 2026-09-17T00:05:38Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7559936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssdiuonvecl8bulup4napa18plli3hl-v5g6h1u7r3y6b9q\", pod=\"router-77b5d8dffb-95c5m\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssdiuonvecl8bulup4napa18plli3hl-v5g6h1u7r3y6b9q/router-77b5d8dffb-95c5m has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-16T23:46:37Z\n Ended: 2026-09-17T00:05:38Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7559936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssdisk0mggfhm9130ika04epdikqlho-s5n1l0v6o6a7b8a\", pod=\"router-55df5c6c57-b4j8r\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssdisk0mggfhm9130ika04epdikqlho-s5n1l0v6o6a7b8a/router-55df5c6c57-b4j8r has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"c064292bb99753cb54ccf19145114d834e717bb77409adb391419e903860e0a5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100354655547559936","occurred_at":"2026-09-16T22:42:23Z","failed_at":"e2e","signature_id":"f2b94e7451cbc051fbf0de3b84883a92be81c7f11c25cc3c65ce003806a07176","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fcc7a0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-jbw245668lwj\" to finish creating: GET https://rp.j7559936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6d2a250e-0bac-4f44-a708-541cb00c701f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6d2a250e-0bac-4f44-a708-541cb00c701f\",\n \"name\": \"6d2a250e-0bac-4f44-a708-541cb00c701f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T23:34:27.803730917Z\",\n \"endTime\": \"2026-09-17T00:03:30.376925674Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m59s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fcc7a0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-jbw245668lwj\" to finish creating: GET https://rp.j7559936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6d2a250e-0bac-4f44-a708-541cb00c701f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6d2a250e-0bac-4f44-a708-541cb00c701f\",\n \"name\": \"6d2a250e-0bac-4f44-a708-541cb00c701f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T23:34:27.803730917Z\",\n \"endTime\": \"2026-09-17T00:03:30.376925674Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m59s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d30c57cc9cda9686c783f2f49182537f80549a358226787f174844b303f6bede","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100354655547559936","occurred_at":"2026-09-16T22:42:23Z","failed_at":"alert","signature_id":"940a5892f0bde8defb7629a002baedf9228b8f92a2cf58838e303561286c491d","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T00:07:06Z\n Ended: 2026-09-17T00:23:04Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7559936-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-554b597df9-2phx7\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-554b597df9-2phx7 (namespace mgmt-agent) on cluster ci01-j7559936-mgmt-2 has averaged 182% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-17T00:07:06Z\n Ended: 2026-09-17T00:26:06Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7559936-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-7ddmw\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-7ddmw (namespace arobit) on cluster ci01-j7559936-mgmt-2 has averaged 200.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100353767223332864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6940,"pr_state":"closed","pr_sha":"62cdd448d25099f3b77adb5f5e21439e857c6d5f","final_merged_sha":"62cdd448d25099f3b77adb5f5e21439e857c6d5f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T22:38:55Z","started_at":"2026-09-16T22:38:55Z","completed_at":"2026-09-17T00:44:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:23.928970967Z","timing_metadata_checked_at":"2026-09-18T15:41:23.928970967Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:23.93254508Z","region_metadata_checked_at":"2026-09-18T15:41:23.93254508Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"51ef229703bc6eb0b450b9ecdb4a130d6879489ced062e9b497c883766398153","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100353767223332864","occurred_at":"2026-09-16T22:38:55Z","failed_at":"alert","signature_id":"e71be1b08c965527fccf228721850a63e0d7ab82463979308010bd46b7d5afcc","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 6 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T00:03:50Z\n Ended: 2026-09-17T00:10:50Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3332864-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-2pvnx\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-2pvnx (namespace arobit) on cluster ci01-j3332864-mgmt-1 has averaged 186.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-17T00:03:50Z\n Ended: 2026-09-17T00:10:50Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3332864-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-dd7947c67-r26c4\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-dd7947c67-r26c4 (namespace mgmt-agent) on cluster ci01-j3332864-mgmt-1 has averaged 191.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-17T00:03:50Z\n Ended: 2026-09-17T00:13:50Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3332864-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-dd85c69cd-czx9t\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-dd85c69cd-czx9t (namespace clusters-service) on cluster ci01-j3332864-svc has averaged 238.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-17T00:03:50Z\n Ended: 2026-09-17T00:13:50Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3332864-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-558d75db97-9xbgb\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-558d75db97-9xbgb (namespace aro-hcp) on cluster ci01-j3332864-svc has averaged 416.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-17T00:03:50Z\n Ended: 2026-09-17T00:13:50Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3332864-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-72f4l\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-72f4l (namespace arobit) on cluster ci01-j3332864-svc has averaged 179.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-09-17T00:28:51Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3332864-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-dd85c69cd-vfcn8\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-dd85c69cd-vfcn8 (namespace clusters-service) on cluster ci01-j3332864-svc has averaged 230.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100351996258160640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6894,"pr_state":"closed","pr_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","final_merged_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T22:31:53Z","started_at":"2026-09-16T22:31:53Z","completed_at":"2026-09-17T00:33:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:33.629734378Z","timing_metadata_checked_at":"2026-09-18T15:40:33.629734378Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:33.632588288Z","region_metadata_checked_at":"2026-09-18T15:40:33.632588288Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"a1b241897615028e901390b695a29a9af9c04ecf30c9f7c17a91fe055c2f34b5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100351996258160640","occurred_at":"2026-09-16T22:31:53Z","failed_at":"e2e","signature_id":"6606b623ae18c88ac8a9b2ec7e9a18026cd7fd696af00a64c201797796b23884","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-v9lkhvnzdhld/cluster-candidate-4-21-jrkc8h should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ba8a40\u003e: \n failed to create HCP cluster cluster-candidate-4-21-jrkc8h: failed waiting for cluster=\"cluster-candidate-4-21-jrkc8h\" in resourcegroup=\"rg-candidate-4-21-v9lkhvnzdhld\" to finish creating: GET https://rp.j8160640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/61166157-eac2-4677-a0c1-58bf48e34b38\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/61166157-eac2-4677-a0c1-58bf48e34b38\",\n \"name\": \"61166157-eac2-4677-a0c1-58bf48e34b38\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T23:26:49.985549857Z\",\n \"endTime\": \"2026-09-16T23:55:49.56140206Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m41s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-v9lkhvnzdhld/cluster-candidate-4-21-jrkc8h should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ba8a40\u003e: \n failed to create HCP cluster cluster-candidate-4-21-jrkc8h: failed waiting for cluster=\"cluster-candidate-4-21-jrkc8h\" in resourcegroup=\"rg-candidate-4-21-v9lkhvnzdhld\" to finish creating: GET https://rp.j8160640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/61166157-eac2-4677-a0c1-58bf48e34b38\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/61166157-eac2-4677-a0c1-58bf48e34b38\",\n \"name\": \"61166157-eac2-4677-a0c1-58bf48e34b38\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T23:26:49.985549857Z\",\n \"endTime\": \"2026-09-16T23:55:49.56140206Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m41s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b99e1d2d2ccb4da841897fe7035271f7bbed009e5dd2998239e8fe684e2ff77e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100351996258160640","occurred_at":"2026-09-16T22:31:53Z","failed_at":"alert","signature_id":"fc7a5f2beeb914b7f9d5e98e516ffb3d0e395b2c10277b6a8a41faeeb997a062","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T00:01:43Z\n Ended: 2026-09-17T00:07:42Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j8160640-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-8cxpn\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-8cxpn (namespace arobit) on cluster ci01-j8160640-mgmt-1 has averaged 131.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"e7f0ee735a89d41d5423ceabf33f719afff063e19714388f6a1dcf83414bb09b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100351996258160640","occurred_at":"2026-09-16T22:31:53Z","failed_at":"alert","signature_id":"e2e746b818d8319297b0cfc1ce06bb87bc7386a6e269943bf13e78c9d13237ab","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T23:38:16Z\n Ended: 2026-09-17T00:05:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8160640-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssdff0k383bfd7grp5qnbnt5v4r1b0u-d3b2r2a4t0o4l6a\", pod=\"router-5564cd696f-nb8mv\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssdff0k383bfd7grp5qnbnt5v4r1b0u-d3b2r2a4t0o4l6a/router-5564cd696f-nb8mv has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100351264033345536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6880,"pr_state":"closed","pr_sha":"b0e6adbe8a3e399d6ef90a427d6581372f590570","final_merged_sha":"a20dd280dd6b7499cf853051219487314ef1e460","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T22:22:06Z","started_at":"2026-09-16T22:22:06Z","completed_at":"2026-09-17T00:35:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:30.836822033Z","timing_metadata_checked_at":"2026-09-18T15:40:30.836822033Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:30.840007245Z","region_metadata_checked_at":"2026-09-18T15:40:30.840007245Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"0bd9cb6216ee1bebb873b7730f9087e7aba3d0f90c516842403994c8c286a108","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100351264033345536","occurred_at":"2026-09-16T22:22:06Z","failed_at":"alert","signature_id":"38cba13ec304a2d65c8ec3a2bb0373cae43840dfeeed1f7a55cc2d8c840f0488","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 5 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T23:51:00Z\n Ended: 2026-09-17T00:03:00Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3345536-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-9b88878b8-n2h2f\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-9b88878b8-n2h2f (namespace clusters-service) on cluster ci01-j3345536-svc has averaged 202% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T23:51:00Z\n Ended: 2026-09-17T00:03:00Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3345536-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-rlqtg\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-rlqtg (namespace arobit) on cluster ci01-j3345536-svc has averaged 160.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T23:51:00Z\n Ended: 2026-09-17T00:04:01Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3345536-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-9b88878b8-hnwhg\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-9b88878b8-hnwhg (namespace clusters-service) on cluster ci01-j3345536-svc has averaged 728% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-16T23:51:00Z\n Ended: 2026-09-17T00:03:01Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3345536-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-db5b7c885-ggk6l\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-db5b7c885-ggk6l (namespace aro-hcp) on cluster ci01-j3345536-svc has averaged 379.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-16T23:56:00Z\n Ended: 2026-09-17T00:05:05Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3345536-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-d9q79\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-d9q79 (namespace arobit) on cluster ci01-j3345536-mgmt-1 has averaged 130.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"49c12da767224c6cb188c1e2d61099918f0b6f5f8ff4efe397e77b22f1652883","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100351264033345536","occurred_at":"2026-09-16T22:22:06Z","failed_at":"alert","signature_id":"db6b8a56aaf730fdd696fc547f30dac94889302e62174f2dd2508fcd332898b0","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T23:31:05Z\n Ended: 2026-09-16T23:50:06Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3345536-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssdc08lq86mikr3m7qtfj8ulr4bs703-h7y5g4s0a4a5j6c\", pod=\"router-5bb8b7c8bd-thvsf\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssdc08lq86mikr3m7qtfj8ulr4bs703-h7y5g4s0a4a5j6c/router-5bb8b7c8bd-thvsf has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100351223038218240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6894,"pr_state":"closed","pr_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","final_merged_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T22:21:29Z","started_at":"2026-09-16T22:21:29Z","completed_at":"2026-09-17T00:25:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:23.750996822Z","timing_metadata_checked_at":"2026-09-18T15:41:23.750996822Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:23.755189837Z","region_metadata_checked_at":"2026-09-18T15:41:23.755189837Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"00c42c12b77a95c7f7d5ea5b1db6784f4d0f430dd91acc31bcfdadf39bcead4d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100351223038218240","occurred_at":"2026-09-16T22:21:29Z","failed_at":"alert","signature_id":"fbe220275de66dc82c17bd70379b1968bfca47f26279aaeb942365f7083c7aae","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T23:51:32Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j8218240-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-579c9576fd-sv86s\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-579c9576fd-sv86s (namespace clusters-service) on cluster ci01-j8218240-svc has averaged 751.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T23:51:32Z\n Ended: 2026-09-16T23:57:37Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j8218240-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-x6nxz\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-x6nxz (namespace arobit) on cluster ci01-j8218240-mgmt-2 has averaged 130.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T23:51:32Z\n Ended: 2026-09-16T23:57:37Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j8218240-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-78464c98b7-mqbht\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-78464c98b7-mqbht (namespace mgmt-agent) on cluster ci01-j8218240-mgmt-2 has averaged 168.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100346022097063936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4878,"pr_state":"closed","pr_sha":"8a73d0dda0fc968a36e83392cf2227e915acddc9","final_merged_sha":"244d0c16bcabe8284755cf089f7bb166cd38a435","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T22:08:08Z","started_at":"2026-09-16T22:08:08Z","completed_at":"2026-09-16T23:52:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:39:59.480707911Z","timing_metadata_checked_at":"2026-09-18T15:39:59.480707911Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:39:59.484813125Z","region_metadata_checked_at":"2026-09-18T15:39:59.484813125Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"6920450b1ddbec0c0fc170fec20d110f5c2ce914aa9b435a5e3477aa7797a3f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100346022097063936","occurred_at":"2026-09-16T22:08:08Z","failed_at":"e2e","signature_id":"40918e34615df9a134d5bd8ce9ad3cec41e9fc04a925911878e36407f747a316","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f12a00\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-mt67hxmh7jr7\" to finish creating: GET https://rp.j7063936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2d6116a-8dc5-497c-a1b9-37dbf3d9da56\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2d6116a-8dc5-497c-a1b9-37dbf3d9da56\",\n \"name\": \"f2d6116a-8dc5-497c-a1b9-37dbf3d9da56\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T22:42:15.998420325Z\",\n \"endTime\": \"2026-09-16T23:11:18.627710941Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m7s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f12a00\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-mt67hxmh7jr7\" to finish creating: GET https://rp.j7063936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2d6116a-8dc5-497c-a1b9-37dbf3d9da56\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2d6116a-8dc5-497c-a1b9-37dbf3d9da56\",\n \"name\": \"f2d6116a-8dc5-497c-a1b9-37dbf3d9da56\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T22:42:15.998420325Z\",\n \"endTime\": \"2026-09-16T23:11:18.627710941Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m7s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a39d7c341922077c873e124a43f81eed4a60877fc3996bf66551eda86a2e56ad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100346022097063936","occurred_at":"2026-09-16T22:08:08Z","failed_at":"alert","signature_id":"5098cde8f56024f9b94cf40f2a466897158f3f3994b796f3a58cc2d52fcb7504","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 14 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T23:11:46Z\n Ended: 2026-09-16T23:30:45Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7063936-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-chksz\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-chksz (namespace arobit) on cluster ci01-j7063936-mgmt-1 has averaged 159.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-16T23:11:46Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7063936-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-5t5f6\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-5t5f6 (namespace arobit) on cluster ci01-j7063936-mgmt-2 has averaged 191.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T23:11:46Z\n Ended: 2026-09-16T23:36:46Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7063936-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-554565c5c6-qnkg5\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-554565c5c6-qnkg5 (namespace mgmt-agent) on cluster ci01-j7063936-mgmt-2 has averaged 207.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-16T23:11:46Z\n Ended: 2026-09-16T23:38:45Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7063936-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-557577f7c8-v7lhp\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-557577f7c8-v7lhp (namespace clusters-service) on cluster ci01-j7063936-svc has averaged 241.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-16T23:11:46Z\n Ended: 2026-09-16T23:35:44Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7063936-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-jqspw\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-jqspw (namespace arobit) on cluster ci01-j7063936-svc has averaged 172.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Resolved\n Started: 2026-09-16T23:11:46Z\n Ended: 2026-09-16T23:37:46Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7063936-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-6wt6n\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-6wt6n (namespace arobit) on cluster ci01-j7063936-mgmt-2 has averaged 203.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-09-16T23:11:46Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7063936-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-557577f7c8-tk25g\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-557577f7c8-tk25g (namespace clusters-service) on cluster ci01-j7063936-svc has averaged 780.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 8:\n State: Resolved\n Started: 2026-09-16T23:11:46Z\n Ended: 2026-09-16T23:30:45Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7063936-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-554565c5c6-hz6w6\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-554565c5c6-hz6w6 (namespace mgmt-agent) on cluster ci01-j7063936-mgmt-1 has averaged 159.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 9:\n State: Fired (not resolved)\n Started: 2026-09-16T23:11:46Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7063936-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-79554bdd98-8lr8l\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-79554bdd98-8lr8l (namespace aro-hcp) on cluster ci01-j7063936-svc has averaged 426.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 10:\n State: Resolved\n Started: 2026-09-16T23:11:46Z\n Ended: 2026-09-16T23:38:45Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7063936-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-557577f7c8-wdvtv\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-557577f7c8-wdvtv (namespace clusters-service) on cluster ci01-j7063936-svc has averaged 239.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 11:\n State: Resolved\n Started: 2026-09-16T23:12:45Z\n Ended: 2026-09-16T23:29:45Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7063936-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-wnbdn\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-wnbdn (namespace arobit) on cluster ci01-j7063936-mgmt-2 has averaged 136.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 12:\n State: Resolved\n Started: 2026-09-16T23:17:46Z\n Ended: 2026-09-16T23:23:47Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7063936-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-wxbs2\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-wxbs2 (namespace arobit) on cluster ci01-j7063936-svc has averaged 121.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 13:\n State: Resolved\n Started: 2026-09-16T23:20:46Z\n Ended: 2026-09-16T23:30:45Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7063936-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-pfwsx\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-pfwsx (namespace arobit) on cluster ci01-j7063936-mgmt-1 has averaged 123.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 14:\n State: Resolved\n Started: 2026-09-16T23:21:44Z\n Ended: 2026-09-16T23:28:46Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7063936-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-rtmxc\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-rtmxc (namespace arobit) on cluster ci01-j7063936-mgmt-1 has averaged 121.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"e047186d5e24a3fe2cf3faab3541a8de8e0d853e4c40617215d621b69fe8fe1c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100346022097063936","occurred_at":"2026-09-16T22:08:08Z","failed_at":"e2e","signature_id":"be79ce9f9e2a3c3c1a4622c360ca3519703f1be9bdd39e307bbb6ae587836434","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f78b40\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-bj966m2bsjb2\" to finish creating: GET https://rp.j7063936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/66342529-12fa-4da1-96e8-890f665b0b8e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/66342529-12fa-4da1-96e8-890f665b0b8e\",\n \"name\": \"66342529-12fa-4da1-96e8-890f665b0b8e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T22:39:31.439289121Z\",\n \"endTime\": \"2026-09-16T23:08:37.208910305Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 17m24s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f78b40\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-bj966m2bsjb2\" to finish creating: GET https://rp.j7063936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/66342529-12fa-4da1-96e8-890f665b0b8e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/66342529-12fa-4da1-96e8-890f665b0b8e\",\n \"name\": \"66342529-12fa-4da1-96e8-890f665b0b8e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T22:39:31.439289121Z\",\n \"endTime\": \"2026-09-16T23:08:37.208910305Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 17m24s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100344321164185600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6945,"pr_state":"closed","pr_sha":"344450021ef1cd34320edd7790bea47a65b8b30e","final_merged_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T22:01:23Z","started_at":"2026-09-16T22:01:23Z","completed_at":"2026-09-16T23:41:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:52.47843149Z","timing_metadata_checked_at":"2026-09-18T15:40:52.47843149Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:52.482791406Z","region_metadata_checked_at":"2026-09-18T15:40:52.482791406Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"4d42ead5cf8fd273bbb3bb0374491f49781ece91310ac14676b4ceebda25d1d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100344321164185600","occurred_at":"2026-09-16T22:01:23Z","failed_at":"alert","signature_id":"600a7515642ad65bbfc931d4dd2a57d1a3c95b9594a86121e772644509ea9e97","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 8 time(s) (2 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T22:40:20Z\n Ended: 2026-09-16T22:57:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4185600-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssckc0qi9obj06817o08c2p0t18e9rl-v20261001\", pod=\"router-7c865bd84c-4zb2h\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssckc0qi9obj06817o08c2p0t18e9rl-v20261001/router-7c865bd84c-4zb2h has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-16T23:03:18Z\n Ended: 2026-09-16T23:10:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4185600-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssckc0qi9obj06817o08c2p0t18e9rl-v20261001\", pod=\"router-7c865bd84c-4zb2h\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssckc0qi9obj06817o08c2p0t18e9rl-v20261001/router-7c865bd84c-4zb2h has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"73fb925b3f7fc085728fd3c1de7883cf91fa8fbcd292f23c52cfe178385fbcbb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100344321164185600","occurred_at":"2026-09-16T22:01:23Z","failed_at":"alert","signature_id":"f7080c270925219a311e60803887fccb7900c43032060412377f7a288ab7c269","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T23:03:25Z\n Ended: 2026-09-16T23:10:27Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4185600-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-57644fb98d-2b9lt\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-57644fb98d-2b9lt (namespace aro-hcp) on cluster ci01-j4185600-svc has averaged 418.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T23:03:25Z\n Ended: 2026-09-16T23:10:28Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4185600-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-6d579cc9b7-wks59\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-6d579cc9b7-wks59 (namespace clusters-service) on cluster ci01-j4185600-svc has averaged 243.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T23:03:25Z\n Ended: 2026-09-16T23:10:28Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4185600-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-cs2rw\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-cs2rw (namespace arobit) on cluster ci01-j4185600-svc has averaged 177.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"8996a69e816530eae0523e903506da52d7a889d6bdc39cb20fe714f4c69eff60","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100344321164185600","occurred_at":"2026-09-16T22:01:23Z","failed_at":"e2e","signature_id":"00790225a1ec5d2f2111abfe85ea313169365d1041de24fbe4fa745473e8e330","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:98]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010967c0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-bfxzxpsnx2jt\" to finish creating: GET https://rp.j4185600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/12c613e0-b15b-45cf-943d-cff1110cd7cf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/12c613e0-b15b-45cf-943d-cff1110cd7cf\",\n \"name\": \"12c613e0-b15b-45cf-943d-cff1110cd7cf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T22:29:03.123193133Z\",\n \"endTime\": \"2026-09-16T22:58:10.913482321Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 24m21s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:98]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010967c0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-bfxzxpsnx2jt\" to finish creating: GET https://rp.j4185600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/12c613e0-b15b-45cf-943d-cff1110cd7cf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/12c613e0-b15b-45cf-943d-cff1110cd7cf\",\n \"name\": \"12c613e0-b15b-45cf-943d-cff1110cd7cf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T22:29:03.123193133Z\",\n \"endTime\": \"2026-09-16T22:58:10.913482321Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 24m21s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100343556559343616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6974,"pr_state":"closed","pr_sha":"ad18c381b0fe07c50afb22f07e5b849b9d525738","final_merged_sha":"fdb1b526d51e98ec4ae2b5a2fc9fae30dbe31d6d","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T21:58:20Z","started_at":"2026-09-16T21:58:20Z","completed_at":"2026-09-16T23:57:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:03.863386152Z","timing_metadata_checked_at":"2026-09-18T15:41:03.863386152Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:03.866362863Z","region_metadata_checked_at":"2026-09-18T15:41:03.866362863Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"bf5a4d0784f951ced3296003562afbdf550ea3ecfed18155e6295e37fda56103","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100343556559343616","occurred_at":"2026-09-16T21:58:20Z","failed_at":"alert","signature_id":"180e6e60fd74213dba13f27436fee7ff492e29f8f76edd5be5c54b377947d936","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 8 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T23:08:05Z\n Ended: 2026-09-16T23:20:04Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9343616-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-68bbb68cfb-4n82q\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-68bbb68cfb-4n82q (namespace clusters-service) on cluster ci01-j9343616-svc has averaged 685.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T23:09:03Z\n Ended: 2026-09-16T23:21:04Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9343616-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-68bbb68cfb-dlbqw\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-68bbb68cfb-dlbqw (namespace clusters-service) on cluster ci01-j9343616-svc has averaged 214.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T23:11:06Z\n Ended: 2026-09-16T23:24:05Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9343616-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-b4rjl\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-b4rjl (namespace arobit) on cluster ci01-j9343616-mgmt-1 has averaged 217.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-16T23:11:06Z\n Ended: 2026-09-16T23:22:07Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9343616-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-6bc4f4cb68-nlpz6\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-6bc4f4cb68-nlpz6 (namespace mgmt-agent) on cluster ci01-j9343616-mgmt-2 has averaged 141.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-16T23:13:03Z\n Ended: 2026-09-16T23:29:04Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9343616-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-68bbb68cfb-5jm25\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-68bbb68cfb-5jm25 (namespace clusters-service) on cluster ci01-j9343616-svc has averaged 228.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Resolved\n Started: 2026-09-16T23:13:03Z\n Ended: 2026-09-16T23:29:05Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9343616-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-fnbdj\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-fnbdj (namespace arobit) on cluster ci01-j9343616-svc has averaged 163.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 7:\n State: Resolved\n Started: 2026-09-16T23:13:03Z\n Ended: 2026-09-16T23:29:06Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9343616-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-59d56584b6-7vtjg\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-59d56584b6-7vtjg (namespace aro-hcp) on cluster ci01-j9343616-svc has averaged 395.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 8:\n State: Resolved\n Started: 2026-09-16T23:13:03Z\n Ended: 2026-09-16T23:35:06Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9343616-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-6bc4f4cb68-dmrjw\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-6bc4f4cb68-dmrjw (namespace mgmt-agent) on cluster ci01-j9343616-mgmt-1 has averaged 182.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100335879556960256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6977,"pr_state":"closed","pr_sha":"557c04379e033b9de0bd85904b49103f2cdef1b1","final_merged_sha":"557c04379e033b9de0bd85904b49103f2cdef1b1","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T21:27:50Z","started_at":"2026-09-16T21:27:50Z","completed_at":"2026-09-16T23:15:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:06.471030542Z","timing_metadata_checked_at":"2026-09-18T15:41:06.471030542Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:06.474307054Z","region_metadata_checked_at":"2026-09-18T15:41:06.474307054Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"168017296e16dcdfc0958b133067226169267e248b2bdde3b2c45321380dfa0c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100335879556960256","occurred_at":"2026-09-16T21:27:50Z","failed_at":"alert","signature_id":"71b22a11302a00be21922c71aa067e721f76475093f648235ac53a36e12bcb39","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T22:36:48Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6960256-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-64cc5944b6-97trg\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-64cc5944b6-97trg (namespace clusters-service) on cluster ci01-j6960256-svc has averaged 807.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T22:36:48Z\n Ended: 2026-09-16T22:50:46Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6960256-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-5v5zv\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-5v5zv (namespace arobit) on cluster ci01-j6960256-mgmt-1 has averaged 130.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T22:36:48Z\n Ended: 2026-09-16T22:47:50Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6960256-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-5cbb7b949c-wd62b\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-5cbb7b949c-wd62b (namespace mgmt-agent) on cluster ci01-j6960256-mgmt-2 has averaged 139.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-16T22:36:48Z\n Ended: 2026-09-16T22:49:47Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6960256-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-w9rm6\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-w9rm6 (namespace arobit) on cluster ci01-j6960256-mgmt-2 has averaged 156.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"1f0668e0c61568579705cd822078c30077c09a10e5f7711963f4b38b9d643ba5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100335879556960256","occurred_at":"2026-09-16T21:27:50Z","failed_at":"alert","signature_id":"c599c77a6372b3affcde70e85dd51dd5124ab9de3cabd625edc84ba8a133bbfb","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T22:12:55Z\n Ended: 2026-09-16T22:18:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6960256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssc7miveafqg7ekrqti3nqvne3cgsn6-m6t6u0i7x2h4r2q\", pod=\"router-5f6867d758-4rblw\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssc7miveafqg7ekrqti3nqvne3cgsn6-m6t6u0i7x2h4r2q/router-5f6867d758-4rblw has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100335557920952320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6856,"pr_state":"closed","pr_sha":"3488ce75b90ddf477eaba7e960e23f8d4d5145bc","final_merged_sha":"96f38fd5a277adea0aeec323ef5685124af19024","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T21:26:33Z","started_at":"2026-09-16T21:26:33Z","completed_at":"2026-09-16T23:40:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:28.695326932Z","timing_metadata_checked_at":"2026-09-18T15:40:28.695326932Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:28.698335242Z","region_metadata_checked_at":"2026-09-18T15:40:28.698335242Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"e7fbeeae79a2a48737c4a9aebcde8d4c5bd9b75aba96958576735694549762f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100335557920952320","occurred_at":"2026-09-16T21:26:33Z","failed_at":"alert","signature_id":"ceaa91f4efdcdb0d1c542d52b327737fcf8ae4ccbf48f0c17aab82acbcc99238","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T22:57:46Z\n Ended: 2026-09-16T23:15:48Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0952320-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-5984h\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-5984h (namespace arobit) on cluster ci01-j0952320-mgmt-2 has averaged 241.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T22:57:46Z\n Ended: 2026-09-16T23:04:47Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0952320-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-gr7jc\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-gr7jc (namespace arobit) on cluster ci01-j0952320-mgmt-2 has averaged 146.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T22:57:46Z\n Ended: 2026-09-16T23:05:46Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0952320-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-qff9w\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-qff9w (namespace arobit) on cluster ci01-j0952320-mgmt-2 has averaged 124.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-16T22:57:46Z\n Ended: 2026-09-16T23:14:48Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0952320-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-78ff4b7d87-grnwc\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-78ff4b7d87-grnwc (namespace mgmt-agent) on cluster ci01-j0952320-mgmt-2 has averaged 247.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100332359231475712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6820,"pr_state":"open","pr_sha":"c99d95a578f89e3e4c3bfc476a031bb4b59056f4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T21:13:51Z","started_at":"2026-09-16T21:13:51Z","completed_at":"2026-09-16T22:15:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:25.610002431Z","timing_metadata_checked_at":"2026-09-18T15:40:25.610002431Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:25.614311647Z","region_metadata_checked_at":"2026-09-18T15:40:25.614311647Z"},"failed_at":["alert","provision"],"failed_test_count":4,"occurrences":[{"row_id":"3eae2392ba3449221ad945015f59cbda45e88ee2bb1168fa13db2c2bd96ff024","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100332359231475712","occurred_at":"2026-09-16T21:13:51Z","failed_at":"provision","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","test_suite":"step graph","failure_text":"time=2026-09-16T21:39:23.882Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-16T21:39:24.651Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-09-16T21:39:25.419Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=1957e733d563fe8d01011d2966a166813e9314307467a323647f090022c9d7a7 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1475712-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F1957e733d563fe8d01011d2966a166813e9314307467a323647f090022c9d7a7\ntime=2026-09-16T22:09:23.556Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded"}},{"row_id":"70b5be67d4e31f9c1bd1a11b18c4ff55cf2a047979577298645332f1ab6b9008","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100332359231475712","occurred_at":"2026-09-16T21:13:51Z","failed_at":"alert","signature_id":"4cfee31556334d3f7789166186ab36ba11bfce4b40f49a38dd4a3b6152068d51","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T21:55:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1475712-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-dgkdj\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-dgkdj has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-16T21:55:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1475712-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-cb282\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-cb282 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-16T21:55:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1475712-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-hdnbp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-hdnbp has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-16T21:55:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1475712-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-f58cs\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-f58cs has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"80541be17bb612085662f949718e9ffb5eb29ac1fe9d8026c11957dd47332526","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100332359231475712","occurred_at":"2026-09-16T21:13:51Z","failed_at":"alert","signature_id":"617d8bea63f7c0a8e5aa65e252453013f2f87eaabc747dea5d193edf0ea92b6a","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T22:06:04Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j1475712-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.104:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1475712/providers/microsoft.monitor/accounts/services-j1475712\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5979b8454d-59mfc\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-16T22:06:04Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j1475712-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.104:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1475712/providers/microsoft.monitor/accounts/services-j1475712\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5979b8454d-59mfc\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired"}},{"row_id":"c36b0c0ac0c7e1d6e3910e03fe0f858b0a9bdb155789465dd9c320ba9f7e368c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100332359231475712","occurred_at":"2026-09-16T21:13:51Z","failed_at":"alert","signature_id":"9b1a4c0162aa96d1ba2cfc1a58fe8450105df2f95a84e56b0c0206685fdb86fb","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T22:05:46Z\n Severity: Sev3\n Labels: alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j1475712-svc\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"warning\"\n Description: The Arobit forwarder metrics endpoint on cluster ci01-j1475712-svc has been unreachable for 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100331448274784256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6752,"pr_state":"closed","pr_sha":"d433c21668cfdc66f98262f9099c8086b4397248","final_merged_sha":"d433c21668cfdc66f98262f9099c8086b4397248","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T21:10:14Z","started_at":"2026-09-16T21:10:14Z","completed_at":"2026-09-16T22:54:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:20.244288826Z","timing_metadata_checked_at":"2026-09-18T15:40:20.244288826Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:20.247789239Z","region_metadata_checked_at":"2026-09-18T15:40:20.247789239Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"84a3ed86d3aabd05f67b620b825a5de3d967166de4b8b9683af27e83a12edc65","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100331448274784256","occurred_at":"2026-09-16T21:10:14Z","failed_at":"alert","signature_id":"30e6e0d7b43f0480d6e10d35a84534dd048930cbdf424724d15cc0834916efb3","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 5 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T22:15:49Z\n Ended: 2026-09-16T22:30:50Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4784256-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-5ff76f7dfd-8gkdt\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-5ff76f7dfd-8gkdt (namespace mgmt-agent) on cluster ci01-j4784256-mgmt-1 has averaged 179.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T22:15:49Z\n Ended: 2026-09-16T22:31:51Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4784256-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-6m54g\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-6m54g (namespace arobit) on cluster ci01-j4784256-mgmt-2 has averaged 143.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T22:15:49Z\n Ended: 2026-09-16T22:37:50Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4784256-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-7bfb85644-ctddh\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-7bfb85644-ctddh (namespace clusters-service) on cluster ci01-j4784256-svc has averaged 238.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-16T22:15:49Z\n Ended: 2026-09-16T22:27:54Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4784256-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-jts9j\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-jts9j (namespace arobit) on cluster ci01-j4784256-mgmt-2 has averaged 255.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-16T22:15:49Z\n Ended: 2026-09-16T22:41:49Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4784256-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-7bfb85644-6sh92\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-7bfb85644-6sh92 (namespace clusters-service) on cluster ci01-j4784256-svc has averaged 287.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100328454825185280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6980,"pr_state":"closed","pr_sha":"4c6ef86c4410d1e48dd7653c177a4370cf38a246","final_merged_sha":"4c6ef86c4410d1e48dd7653c177a4370cf38a246","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T20:58:20Z","started_at":"2026-09-16T20:58:20Z","completed_at":"2026-09-16T22:45:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:07.795598812Z","timing_metadata_checked_at":"2026-09-18T15:41:07.795598812Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:07.799536226Z","region_metadata_checked_at":"2026-09-18T15:41:07.799536226Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"9f0021cf2954940628ce127f4187c5158144349931e030fc3ab1a1b899364c3d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100328454825185280","occurred_at":"2026-09-16T20:58:20Z","failed_at":"alert","signature_id":"cfb1506b480d229252a933ef91e34400b8d02a4638f13cabb01dc1d4175a17b8","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T22:02:06Z\n Ended: 2026-09-16T22:09:07Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5185280-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-7bb74f5747-b8mnf\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-7bb74f5747-b8mnf (namespace mgmt-agent) on cluster ci01-j5185280-mgmt-2 has averaged 158.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T22:02:06Z\n Ended: 2026-09-16T22:08:06Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5185280-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-68b58f9664-thkdd\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-68b58f9664-thkdd (namespace clusters-service) on cluster ci01-j5185280-svc has averaged 247.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T22:02:07Z\n Ended: 2026-09-16T22:09:07Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5185280-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-n5mgg\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-n5mgg (namespace arobit) on cluster ci01-j5185280-mgmt-2 has averaged 146.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100327783220645888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6966,"pr_state":"closed","pr_sha":"c93ff943c35ad2d21e3a98df5a27db691c9749a7","final_merged_sha":"c93ff943c35ad2d21e3a98df5a27db691c9749a7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T20:55:40Z","started_at":"2026-09-16T20:55:40Z","completed_at":"2026-09-16T22:53:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:00.43852802Z","timing_metadata_checked_at":"2026-09-18T15:41:00.43852802Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:00.443486138Z","region_metadata_checked_at":"2026-09-18T15:41:00.443486138Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"0692b7e4df0967726792239e5db86dffb2f653571fc1bd3435e44bce67b31e9f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100327783220645888","occurred_at":"2026-09-16T20:55:40Z","failed_at":"alert","signature_id":"18e89a273ad3f5766b2dd6e52e34589ba49632a0e92e5d7b81de104647e14a35","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T22:06:02Z\n Ended: 2026-09-16T22:32:59Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0645888-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-55fbb6c9-ccknz\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-55fbb6c9-ccknz (namespace aro-hcp) on cluster ci01-j0645888-svc has averaged 419.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T22:06:02Z\n Ended: 2026-09-16T22:32:59Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0645888-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-586c94d488-qmqpw\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-586c94d488-qmqpw (namespace clusters-service) on cluster ci01-j0645888-svc has averaged 776.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T22:06:02Z\n Ended: 2026-09-16T22:20:00Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0645888-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-586c94d488-kqxfq\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-586c94d488-kqxfq (namespace clusters-service) on cluster ci01-j0645888-svc has averaged 232.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-16T22:06:02Z\n Ended: 2026-09-16T22:29:59Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0645888-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-42shp\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-42shp (namespace arobit) on cluster ci01-j0645888-svc has averaged 224% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"14cb8ad302ed3e26822b10283ef7506fe5de19aec6e182fc1ef008187bd7ecaf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100327783220645888","occurred_at":"2026-09-16T20:55:40Z","failed_at":"alert","signature_id":"4c70e8e6dfdf51fe895bb78044f498f38b34d7b31ddb973e4ed72258e72ee568","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T21:54:30Z\n Ended: 2026-09-16T22:00:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0645888-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssbr3b0dohpb942farb5mg8qfbn5fpn-v9m4k2a5i1j6u4q\", pod=\"openshift-apiserver-698dfb475d-rm5kz\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssbr3b0dohpb942farb5mg8qfbn5fpn-v9m4k2a5i1j6u4q/openshift-apiserver-698dfb475d-rm5kz has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"b368522ea717841a0bf803ebbe3e8dc7455b6e5e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T20:42:32Z","started_at":"2026-09-16T20:42:32Z","completed_at":"2026-09-16T22:32:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:05.627634324Z","timing_metadata_checked_at":"2026-09-18T15:40:05.627634324Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:05.63481855Z","region_metadata_checked_at":"2026-09-18T15:40:05.63481855Z"},"failed_at":["alert","e2e"],"failed_test_count":12,"occurrences":[{"row_id":"034ff9f8cbb9ce7d794397c5cd5284692a3e76e9ad5d1ac18c3c6d8a2b5e8e63","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","failed_at":"alert","signature_id":"d21baacf92b823fbe47d168aff240c5f90998a22314dd5cc365610b36ceba21b","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 11 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T21:35:38Z\n Ended: 2026-09-16T22:04:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3686400-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssbld7o605mco1c6q9ktqjhs93eqe6s-j8l1v5a2x0c6s4f\", pod=\"router-6bcf688cbb-nxnjs\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssbld7o605mco1c6q9ktqjhs93eqe6s-j8l1v5a2x0c6s4f/router-6bcf688cbb-nxnjs has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-16T21:35:38Z\n Ended: 2026-09-16T21:59:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3686400-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssbl8nc6r8al7si3mq1rjgliso80a2c-ephemeral-disk\", pod=\"router-6b8476f84b-vhtcn\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssbl8nc6r8al7si3mq1rjgliso80a2c-ephemeral-disk/router-6b8476f84b-vhtcn has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-16T21:36:40Z\n Ended: 2026-09-16T22:03:38Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3686400-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssbl0m0uhvtufc2gt2visu064e21mub-u5h1i4b8r4n0e8j\", pod=\"router-7b47b96958-mgtjv\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssbl0m0uhvtufc2gt2visu064e21mub-u5h1i4b8r4n0e8j/router-7b47b96958-mgtjv has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-16T21:36:40Z\n Ended: 2026-09-16T21:59:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3686400-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssbl4s13qkvde5phnn7127rdghp4v37-t0q4t2z8c3u3y8r\", pod=\"router-58bf477cdf-kpghv\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssbl4s13qkvde5phnn7127rdghp4v37-t0q4t2z8c3u3y8r/router-58bf477cdf-kpghv has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-16T21:36:40Z\n Ended: 2026-09-16T22:07:38Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3686400-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssbl7klscstpk49ij7po527rbns6pgi-p3b0r5x5e4p9u1v\", pod=\"router-5745db7ddf-xsgg8\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssbl7klscstpk49ij7po527rbns6pgi-p3b0r5x5e4p9u1v/router-5745db7ddf-xsgg8 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-16T21:36:40Z\n Ended: 2026-09-16T22:02:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3686400-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssbl966ue7bj7p60s8sinf6qmbqnuok-h7n4x2g5p8k0m4i\", pod=\"router-5578b5cd64-55cjc\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssbl966ue7bj7p60s8sinf6qmbqnuok-h7n4x2g5p8k0m4i/router-5578b5cd64-55cjc has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-16T21:36:40Z\n Ended: 2026-09-16T21:59:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3686400-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssbl28cqnaa6fvtqs8lmbcrn1lmg4a5-b7b7m9f6p3w4f2s\", pod=\"router-69d54c7db5-fvcnx\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssbl28cqnaa6fvtqs8lmbcrn1lmg4a5-b7b7m9f6p3w4f2s/router-69d54c7db5-fvcnx has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-16T21:36:40Z\n Ended: 2026-09-16T22:07:38Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3686400-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssbl7bvf4r5c6jetumc9dq8tqgsfr18-cilium-cl\", pod=\"router-7b8cc969f7-4pr9f\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssbl7bvf4r5c6jetumc9dq8tqgsfr18-cilium-cl/router-7b8cc969f7-4pr9f has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-16T21:36:40Z\n Ended: 2026-09-16T21:59:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3686400-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssbl81hl56coj0npht1eplmr0rhedd6-d1c8e6y4l1z3b4z\", pod=\"router-654684649f-ztg2t\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssbl81hl56coj0npht1eplmr0rhedd6-d1c8e6y4l1z3b4z/router-654684649f-ztg2t has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-16T21:36:40Z\n Ended: 2026-09-16T21:58:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3686400-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssbl5466uev4mojug68na19t1k1bhpg-j2c1q0a4t2i8o4k\", pod=\"router-75c68fb5b9-cfmtm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssbl5466uev4mojug68na19t1k1bhpg-j2c1q0a4t2i8o4k/router-75c68fb5b9-cfmtm has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-16T21:36:40Z\n Ended: 2026-09-16T22:05:39Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3686400-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssbl0c4vr90is2erighv55thdli5lnq-m4n7d3j0c0g6t2h\", pod=\"router-86c7ffd6cc-mhnb6\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssbl0c4vr90is2erighv55thdli5lnq-m4n7d3j0c0g6t2h/router-86c7ffd6cc-mhnb6 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"0f2eea2aa1d725e2cdf95b24dc97d11bb27d0e5fbaedfa5e57c231f9d7c334de","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","failed_at":"e2e","signature_id":"fc5496eac86ff2a1981e1d0213c04f2b8c1f98c1155641ca8fa469251820f45a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-n9lbrg2g9s97/cluster-candidate-5-0-qbdt9b should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fac960\u003e: \n failed to create HCP cluster cluster-candidate-5-0-qbdt9b: failed waiting for cluster=\"cluster-candidate-5-0-qbdt9b\" in resourcegroup=\"rg-candidate-5-0-n9lbrg2g9s97\" to finish creating: GET https://rp.j3686400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1d8dfefa-03ec-411c-ae11-a19a19b9c715\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1d8dfefa-03ec-411c-ae11-a19a19b9c715\",\n \"name\": \"1d8dfefa-03ec-411c-ae11-a19a19b9c715\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:22:26.135723838Z\",\n \"endTime\": \"2026-09-16T21:51:33.47561355Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.2 is Partial (want Completed), started 21m49s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-n9lbrg2g9s97/cluster-candidate-5-0-qbdt9b should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fac960\u003e: \n failed to create HCP cluster cluster-candidate-5-0-qbdt9b: failed waiting for cluster=\"cluster-candidate-5-0-qbdt9b\" in resourcegroup=\"rg-candidate-5-0-n9lbrg2g9s97\" to finish creating: GET https://rp.j3686400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1d8dfefa-03ec-411c-ae11-a19a19b9c715\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1d8dfefa-03ec-411c-ae11-a19a19b9c715\",\n \"name\": \"1d8dfefa-03ec-411c-ae11-a19a19b9c715\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:22:26.135723838Z\",\n \"endTime\": \"2026-09-16T21:51:33.47561355Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.2 is Partial (want Completed), started 21m49s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6eb6eb93f827b82eeb37bb1a73fd55d13f51bab84a0eba8f8fdbed8634e28865","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","failed_at":"e2e","signature_id":"79d660f1bacf21e2718e2b4460bbd751ebcedbde1fed3611a2e8b06be51c9b26","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: failed to wait for second cluster \"basic-hcp-cluster2\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0013ea750\u003e: \n GET https://rp.j3686400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd13f769-88f3-4db4-9a16-30099a1f1771\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd13f769-88f3-4db4-9a16-30099a1f1771\",\n \"name\": \"dd13f769-88f3-4db4-9a16-30099a1f1771\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:22:59.852105636Z\",\n \"endTime\": \"2026-09-16T21:52:03.393836187Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m38s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: failed to wait for second cluster \"basic-hcp-cluster2\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0013ea750\u003e: \n GET https://rp.j3686400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd13f769-88f3-4db4-9a16-30099a1f1771\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd13f769-88f3-4db4-9a16-30099a1f1771\",\n \"name\": \"dd13f769-88f3-4db4-9a16-30099a1f1771\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:22:59.852105636Z\",\n \"endTime\": \"2026-09-16T21:52:03.393836187Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m38s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8cd4e8f8d245197a9b58a3bd1fc5be17fe31287aa5c331d3cf83a4ee8cab68f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","failed_at":"e2e","signature_id":"3728e9e0a9b25fc2b96787f8795446de81019e99ef3c6a67f320e9809b7bdae3","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:163]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000614ae0\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-nbncrnhpbr7g\" to finish creating: GET https://rp.j3686400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e5d0092-fd65-4a3b-8b4d-9aa58e610368\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e5d0092-fd65-4a3b-8b4d-9aa58e610368\",\n \"name\": \"2e5d0092-fd65-4a3b-8b4d-9aa58e610368\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:22:37.508203008Z\",\n \"endTime\": \"2026-09-16T21:51:43.476984266Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m0s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:163]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000614ae0\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-nbncrnhpbr7g\" to finish creating: GET https://rp.j3686400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e5d0092-fd65-4a3b-8b4d-9aa58e610368\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e5d0092-fd65-4a3b-8b4d-9aa58e610368\",\n \"name\": \"2e5d0092-fd65-4a3b-8b4d-9aa58e610368\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:22:37.508203008Z\",\n \"endTime\": \"2026-09-16T21:51:43.476984266Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m0s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"914e0cd1b78c23270fdbafba50330de1f286180613576796d188373f0c007cc3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","failed_at":"alert","signature_id":"29b55dd4d895c72d0d59b9865df950f870325198fd7e112f1858c956d6c3812a","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 7 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T21:56:22Z\n Ended: 2026-09-16T22:07:21Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3686400-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-cnwz7\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-cnwz7 (namespace arobit) on cluster ci01-j3686400-mgmt-2 has averaged 231.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T21:56:22Z\n Ended: 2026-09-16T22:07:21Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3686400-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-66bbcd5d66-jbqlh\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-66bbcd5d66-jbqlh (namespace clusters-service) on cluster ci01-j3686400-svc has averaged 997.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T21:56:22Z\n Ended: 2026-09-16T22:07:20Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3686400-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-67b9d68855-97z44\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-67b9d68855-97z44 (namespace mgmt-agent) on cluster ci01-j3686400-mgmt-2 has averaged 165.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-16T21:56:22Z\n Ended: 2026-09-16T22:22:23Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3686400-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-2sxwz\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-2sxwz (namespace arobit) on cluster ci01-j3686400-svc has averaged 253.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-16T21:57:22Z\n Ended: 2026-09-16T22:07:21Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3686400-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-sxn79\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-sxn79 (namespace arobit) on cluster ci01-j3686400-svc has averaged 137% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Resolved\n Started: 2026-09-16T21:58:21Z\n Ended: 2026-09-16T22:22:22Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3686400-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-66bbcd5d66-l7s2g\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-66bbcd5d66-l7s2g (namespace clusters-service) on cluster ci01-j3686400-svc has averaged 249.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-09-16T21:58:21Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3686400-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-665d88bb7f-br5fz\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-665d88bb7f-br5fz (namespace aro-hcp) on cluster ci01-j3686400-svc has averaged 424.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"984c7bd274c4854b1f6e702130c69c68339bee74c1ce216ee5e40c108b370d6e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:183]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:183]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"ac5227bbeb0c276a16c8aa6e095598a84c7341979673d322f4b504418d14cfca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","failed_at":"e2e","signature_id":"2698bc1670e697e5f02bc62eaeda2db20989111b43c6c7fcc9af92e68d70c247","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:89]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c1e780\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-pzvkp84k7zw9\" to finish creating: GET https://rp.j3686400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9cfbae6c-9a9c-457e-badc-6962e84d4e6c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9cfbae6c-9a9c-457e-badc-6962e84d4e6c\",\n \"name\": \"9cfbae6c-9a9c-457e-badc-6962e84d4e6c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:22:37.245133028Z\",\n \"endTime\": \"2026-09-16T21:51:43.591474051Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m35s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:89]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c1e780\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-pzvkp84k7zw9\" to finish creating: GET https://rp.j3686400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9cfbae6c-9a9c-457e-badc-6962e84d4e6c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9cfbae6c-9a9c-457e-badc-6962e84d4e6c\",\n \"name\": \"9cfbae6c-9a9c-457e-badc-6962e84d4e6c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:22:37.245133028Z\",\n \"endTime\": \"2026-09-16T21:51:43.591474051Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m35s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bcb74a57f8beab803dcce892ec750c5ae042baa1c8212c29e877edc1b622c07e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","failed_at":"e2e","signature_id":"0f792fe0731eefcb63f38f15bca9a1aa450cefba466c1d19cff0ae8ed18fb263","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:99]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aa2500\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-87r4ztg6n9l7\" to finish creating: GET https://rp.j3686400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d00f648b-166f-4d0d-b942-191a0be88942\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d00f648b-166f-4d0d-b942-191a0be88942\",\n \"name\": \"d00f648b-166f-4d0d-b942-191a0be88942\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:22:41.289177596Z\",\n \"endTime\": \"2026-09-16T21:51:43.506899271Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m16s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:99]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aa2500\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-87r4ztg6n9l7\" to finish creating: GET https://rp.j3686400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d00f648b-166f-4d0d-b942-191a0be88942\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d00f648b-166f-4d0d-b942-191a0be88942\",\n \"name\": \"d00f648b-166f-4d0d-b942-191a0be88942\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:22:41.289177596Z\",\n \"endTime\": \"2026-09-16T21:51:43.506899271Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m16s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d370f4191050dbb8b79e29dbd19bf13ddcab67c6b04bfd6adfa572d179f568e5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","failed_at":"e2e","signature_id":"a52a7332954a9dce3940383eb5861ec4596c82a180f58ab40695aaa3094465e8","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:124]: failed to create HCP cluster \"cluster-zstream-5-0-9bj8v9\" with version 5.0.0-rc.1 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aaa440\u003e: \n failed to create HCP cluster cluster-zstream-5-0-9bj8v9: failed waiting for cluster=\"cluster-zstream-5-0-9bj8v9\" in resourcegroup=\"rg-zstream-upgrade-5-0-km48vzh4hvjm\" to finish creating: GET https://rp.j3686400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd9c2ad0-6041-47d2-bcf5-57e68c17fd53\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd9c2ad0-6041-47d2-bcf5-57e68c17fd53\",\n \"name\": \"bd9c2ad0-6041-47d2-bcf5-57e68c17fd53\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:22:40.065777395Z\",\n \"endTime\": \"2026-09-16T21:51:43.490166655Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.1 is Partial (want Completed), started 22m40s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:124]: failed to create HCP cluster \"cluster-zstream-5-0-9bj8v9\" with version 5.0.0-rc.1 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aaa440\u003e: \n failed to create HCP cluster cluster-zstream-5-0-9bj8v9: failed waiting for cluster=\"cluster-zstream-5-0-9bj8v9\" in resourcegroup=\"rg-zstream-upgrade-5-0-km48vzh4hvjm\" to finish creating: GET https://rp.j3686400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd9c2ad0-6041-47d2-bcf5-57e68c17fd53\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd9c2ad0-6041-47d2-bcf5-57e68c17fd53\",\n \"name\": \"bd9c2ad0-6041-47d2-bcf5-57e68c17fd53\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:22:40.065777395Z\",\n \"endTime\": \"2026-09-16T21:51:43.490166655Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.1 is Partial (want Completed), started 22m40s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e815c175eaa1ba54e273016842f0d18f0478b909aa839445b6aa5397c7392ab3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","failed_at":"e2e","signature_id":"2ffe72f7718510ccd07fc00e17f58bc0013af474b7348754fe62e7d3c4440904","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:90]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eaac80\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-5sd5p2zrd22p\" to finish creating: GET https://rp.j3686400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb287309-affe-4cb6-ba60-0ab97101fffb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb287309-affe-4cb6-ba60-0ab97101fffb\",\n \"name\": \"cb287309-affe-4cb6-ba60-0ab97101fffb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:22:15.559693502Z\",\n \"endTime\": \"2026-09-16T21:51:23.369287505Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:90]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eaac80\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-5sd5p2zrd22p\" to finish creating: GET https://rp.j3686400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb287309-affe-4cb6-ba60-0ab97101fffb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb287309-affe-4cb6-ba60-0ab97101fffb\",\n \"name\": \"cb287309-affe-4cb6-ba60-0ab97101fffb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:22:15.559693502Z\",\n \"endTime\": \"2026-09-16T21:51:23.369287505Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"eeb523b0d8b577d30366d0435a2c4b979e30025e456a3b85e635b21adbfe9964","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","failed_at":"e2e","signature_id":"8d9c0886049dce2e52d531fad148d6ed84eeb2446b9625587b295b408c739bc3","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:293]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000adcb40\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-8b5zl2hzsmbj\" to finish creating: GET https://rp.j3686400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/80bc4253-f3f0-43bc-a084-2e6ad2732f13\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/80bc4253-f3f0-43bc-a084-2e6ad2732f13\",\n \"name\": \"80bc4253-f3f0-43bc-a084-2e6ad2732f13\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:22:42.844869164Z\",\n \"endTime\": \"2026-09-16T21:51:43.481872302Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m58s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:293]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000adcb40\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-8b5zl2hzsmbj\" to finish creating: GET https://rp.j3686400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/80bc4253-f3f0-43bc-a084-2e6ad2732f13\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/80bc4253-f3f0-43bc-a084-2e6ad2732f13\",\n \"name\": \"80bc4253-f3f0-43bc-a084-2e6ad2732f13\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:22:42.844869164Z\",\n \"endTime\": \"2026-09-16T21:51:43.481872302Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m58s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f5bb4148c21c8e7ccc053a03ba0234e8aa366d220eaca68a0c2bc42e2395917d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","failed_at":"e2e","signature_id":"50dab7e83b7442f97fb4586fd719c0b5802925b2f02cc972849692995a093dd0","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:85]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001056d00\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-c7nrjx2xztfs\" to finish creating: GET https://rp.j3686400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a484a274-602c-4541-905e-dc02a0a3186b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a484a274-602c-4541-905e-dc02a0a3186b\",\n \"name\": \"a484a274-602c-4541-905e-dc02a0a3186b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:22:10.633199439Z\",\n \"endTime\": \"2026-09-16T21:51:13.479553961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m57s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:85]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001056d00\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-c7nrjx2xztfs\" to finish creating: GET https://rp.j3686400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a484a274-602c-4541-905e-dc02a0a3186b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a484a274-602c-4541-905e-dc02a0a3186b\",\n \"name\": \"a484a274-602c-4541-905e-dc02a0a3186b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:22:10.633199439Z\",\n \"endTime\": \"2026-09-16T21:51:13.479553961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m57s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6805,"pr_state":"closed","pr_sha":"b3bec0e87c8b5d73d493df4173e57e3c3590d2d7","final_merged_sha":"b3bec0e87c8b5d73d493df4173e57e3c3590d2d7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T20:38:26Z","started_at":"2026-09-16T20:38:26Z","completed_at":"2026-09-16T22:15:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:24.477994558Z","timing_metadata_checked_at":"2026-09-18T15:40:24.477994558Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:24.48114127Z","region_metadata_checked_at":"2026-09-18T15:40:24.48114127Z"},"failed_at":["alert","e2e"],"failed_test_count":9,"occurrences":[{"row_id":"08c903b23dd3c5cb8b4a11fa62add9d44d409bda6d2d641042c5742e02cfb4db","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","failed_at":"e2e","signature_id":"6d4635ee801aeb366b63d020c6eeb3ed381519a10bc1a02c8e00a5d443d6550e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-dk8z27cq8m2q/cluster-candidate-5-0-lt55lh should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00088abe0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-lt55lh: failed waiting for cluster=\"cluster-candidate-5-0-lt55lh\" in resourcegroup=\"rg-candidate-5-0-dk8z27cq8m2q\" to finish creating: GET https://rp.j3803776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c5a4fb94-5cbc-41ce-a664-6b95d7517b0f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c5a4fb94-5cbc-41ce-a664-6b95d7517b0f\",\n \"name\": \"c5a4fb94-5cbc-41ce-a664-6b95d7517b0f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:07:59.319124626Z\",\n \"endTime\": \"2026-09-16T21:37:03.975119557Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.2 is Partial (want Completed), started 25m26s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-dk8z27cq8m2q/cluster-candidate-5-0-lt55lh should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00088abe0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-lt55lh: failed waiting for cluster=\"cluster-candidate-5-0-lt55lh\" in resourcegroup=\"rg-candidate-5-0-dk8z27cq8m2q\" to finish creating: GET https://rp.j3803776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c5a4fb94-5cbc-41ce-a664-6b95d7517b0f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c5a4fb94-5cbc-41ce-a664-6b95d7517b0f\",\n \"name\": \"c5a4fb94-5cbc-41ce-a664-6b95d7517b0f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:07:59.319124626Z\",\n \"endTime\": \"2026-09-16T21:37:03.975119557Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.2 is Partial (want Completed), started 25m26s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0ac8352701693208d9b30e689a21ea5b4b3bc6315540477c2eb5eb4005d0e254","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","failed_at":"alert","signature_id":"4ef72abc8b9948fd9bead06d17d6310bcbc699657b6ee721fd16b3106ac79918","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 5 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T21:40:02Z\n Ended: 2026-09-16T21:52:04Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3803776-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-2lktz\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-2lktz (namespace arobit) on cluster ci01-j3803776-mgmt-1 has averaged 137.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T21:40:02Z\n Ended: 2026-09-16T21:54:03Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3803776-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-99f86f48f-d6hvl\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-99f86f48f-d6hvl (namespace mgmt-agent) on cluster ci01-j3803776-mgmt-1 has averaged 164.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-16T21:41:05Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3803776-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-6644cd5d7d-cfxst\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-6644cd5d7d-cfxst (namespace aro-hcp) on cluster ci01-j3803776-svc has averaged 423.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-16T21:41:05Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3803776-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-5454d5ff57-pdkgp\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-5454d5ff57-pdkgp (namespace clusters-service) on cluster ci01-j3803776-svc has averaged 829.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-09-16T21:41:05Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3803776-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-b5mqv\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-b5mqv (namespace arobit) on cluster ci01-j3803776-svc has averaged 235.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"4f682c83e9e9c68011c45b5d2fe439607c0b12e61946531283854a630e6af29e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","failed_at":"alert","signature_id":"0d965c3262ee69273c199442b4bed217de7dd1b47c2e6f6c022895186517f6a2","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T21:44:54Z\n Ended: 2026-09-16T21:51:52Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3803776-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.109:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3803776/providers/microsoft.monitor/accounts/hcps-j3803776\", namespace=\"ocm-arohcpci01-2ssben4sp2jl4ug8rvisqeb9vbtmfdbc-n8l2c7a7y0n1i2r\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-gndsj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ssben4sp2jl4ug8rvisqeb9vbtmfdbc-n8l2c7a7y0n1i2r/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-16T21:45:52Z\n Ended: 2026-09-16T22:01:53Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3803776-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.109:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3803776/providers/microsoft.monitor/accounts/hcps-j3803776\", namespace=\"ocm-arohcpci01-2ssben4sp2jl4ug8rvisqeb9vbtmfdbc-n8l2c7a7y0n1i2r\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-gndsj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ssben4sp2jl4ug8rvisqeb9vbtmfdbc-n8l2c7a7y0n1i2r/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"5d031ee576607c135e10a11958620b2e04cf016023e4051515bfe0200334a0c9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","failed_at":"e2e","signature_id":"64d1fa05ea54883aa2363d63f6b4eef64e006023913f4dea31d5e86c98373938","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n \u003c*errors.joinError | 0xc002923158\u003e: \n VerifyFIPSEnabled failed: failed to check FIPS mode on node o2d9i5v3g7d8o6s-np-1-9rk5g-gzzc5: failed to get pod logs for pod fips-check-psxxs in namespace e2e-fips-test-rx7hl on node o2d9i5v3g7d8o6s-np-1-9rk5g-gzzc5 after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-psxxs)\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n \u003c*errors.joinError | 0xc002923158\u003e: \n VerifyFIPSEnabled failed: failed to check FIPS mode on node o2d9i5v3g7d8o6s-np-1-9rk5g-gzzc5: failed to get pod logs for pod fips-check-psxxs in namespace e2e-fips-test-rx7hl on node o2d9i5v3g7d8o6s-np-1-9rk5g-gzzc5 after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-psxxs)\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f98a5615cab85c5989e6a0ac742f663ddff170e09cb4f716bf9a5978a279cf45","failure_pattern":"VerifyFIPSEnabled failed: failed to check FIPS mode on node \u003cnode\u003e: failed to get pod logs for pod fips-check-psxxs in namespace e2e-fips-test-\u003cid\u003e on node \u003cnode\u003e after retries: an error on the server (\"unknown\") has prevented the request from succeeding","search_query":"VerifyFIPSEnabled failed: failed to check FIPS mode on node o2d9i5v3g7d8o6s-np-1-9rk5g-gzzc5: failed to get pod logs for pod fips-check-psxxs in namespace e2e-fips-test-rx7hl on node o2d9i5v3g7d8o6s-np-1-9rk5g-gzzc5 after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-psxxs)"}},{"row_id":"6351bd8e3f1b36cc83a36c921c2933dd7837ad1e4691a3d41184a75f4ab88b32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","failed_at":"alert","signature_id":"ee56a79fd88c61821c015922fd0060df48fa8ea6f29e86e41766eb43042fab32","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s) (3 unknown, 1 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T21:17:53Z\n Ended: 2026-09-16T21:52:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3803776-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssbec6gci69870qqvi16u5kcapftr0i-j9u1q2u4g1r8a9m\", pod=\"router-74f7d556c5-zbmxx\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssbec6gci69870qqvi16u5kcapftr0i-j9u1q2u4g1r8a9m/router-74f7d556c5-zbmxx has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-16T21:18:51Z\n Ended: 2026-09-16T21:50:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3803776-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssbe8pmlnu9ao5hdglrhgldjpnqc79l-k0m6y3i5a2o3n4m\", pod=\"router-987d55cf6-zxs7p\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssbe8pmlnu9ao5hdglrhgldjpnqc79l-k0m6y3i5a2o3n4m/router-987d55cf6-zxs7p has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-16T21:19:51Z\n Ended: 2026-09-16T21:52:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3803776-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssbeffd09kboib4diqvu4luf8fh3hjb-m7i6y4e0g4j7v8c\", pod=\"router-5fccd44c6d-zgbxq\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssbeffd09kboib4diqvu4luf8fh3hjb-m7i6y4e0g4j7v8c/router-5fccd44c6d-zgbxq has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"9c449f08f5e310bc17c5c576bbddc55e964218c2e7abbb3206cb86302091b97a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","failed_at":"e2e","signature_id":"e798e0c9ad920ab5c7e8728fb7d75c823add0086ed40ac0f3ccf60ed14187ffe","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f849a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-kxnvbkzf9rsh\" to finish creating: GET https://rp.j3803776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56e26800-2105-4cf7-a6ab-3433abf35800\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56e26800-2105-4cf7-a6ab-3433abf35800\",\n \"name\": \"56e26800-2105-4cf7-a6ab-3433abf35800\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:09:44.457930894Z\",\n \"endTime\": \"2026-09-16T21:38:44.111074819Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m11s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f849a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-kxnvbkzf9rsh\" to finish creating: GET https://rp.j3803776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56e26800-2105-4cf7-a6ab-3433abf35800\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56e26800-2105-4cf7-a6ab-3433abf35800\",\n \"name\": \"56e26800-2105-4cf7-a6ab-3433abf35800\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:09:44.457930894Z\",\n \"endTime\": \"2026-09-16T21:38:44.111074819Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m11s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9c93eb3835050e2fe20799ea3fe3cf8b9870ab135b413eee55355f81e2697e8d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","failed_at":"e2e","signature_id":"7037bf26441097d6041fad63a8ad90e3d7ac2c0bc29160794aafceabed76fa0d","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011165a0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-2nsvjcl7j6m6\" to finish creating: GET https://rp.j3803776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a27b380d-13cc-4abe-9932-546cf6483df0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a27b380d-13cc-4abe-9932-546cf6483df0\",\n \"name\": \"a27b380d-13cc-4abe-9932-546cf6483df0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:08:44.335887928Z\",\n \"endTime\": \"2026-09-16T21:37:44.000335858Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011165a0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-2nsvjcl7j6m6\" to finish creating: GET https://rp.j3803776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a27b380d-13cc-4abe-9932-546cf6483df0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a27b380d-13cc-4abe-9932-546cf6483df0\",\n \"name\": \"a27b380d-13cc-4abe-9932-546cf6483df0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:08:44.335887928Z\",\n \"endTime\": \"2026-09-16T21:37:44.000335858Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ac9ec5a213fc0a54eb9ffb2504869909ff5cce59c4e74354109b2503653da088","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","failed_at":"e2e","signature_id":"27af66bf8605d41a0eed60544affb038a78040a481edac1756ed6ae13ca1bce6","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e75200\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-dfv98dd4pvxk\" to finish creating: GET https://rp.j3803776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ad1b2fe-b261-4381-9290-0e87a2df230a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ad1b2fe-b261-4381-9290-0e87a2df230a\",\n \"name\": \"5ad1b2fe-b261-4381-9290-0e87a2df230a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:07:46.906089867Z\",\n \"endTime\": \"2026-09-16T21:36:54.029333318Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 24m25s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e75200\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-dfv98dd4pvxk\" to finish creating: GET https://rp.j3803776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ad1b2fe-b261-4381-9290-0e87a2df230a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ad1b2fe-b261-4381-9290-0e87a2df230a\",\n \"name\": \"5ad1b2fe-b261-4381-9290-0e87a2df230a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:07:46.906089867Z\",\n \"endTime\": \"2026-09-16T21:36:54.029333318Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 24m25s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f7f1017a1b75f9150afacfd19f23219b051865931094836aab5d25aa5d791dd9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","failed_at":"e2e","signature_id":"12b8d48d4627f15facb797beba81c6d3c5baa8cff824ed1576ee1a132af8dff4","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-vtt65b86m6vb/cluster-candidate-4-21-gj8xr2 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c20920\u003e: \n failed to create HCP cluster cluster-candidate-4-21-gj8xr2: failed waiting for cluster=\"cluster-candidate-4-21-gj8xr2\" in resourcegroup=\"rg-candidate-4-21-vtt65b86m6vb\" to finish creating: GET https://rp.j3803776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9bcaf370-2958-4c5f-97cb-f96058b55495\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9bcaf370-2958-4c5f-97cb-f96058b55495\",\n \"name\": \"9bcaf370-2958-4c5f-97cb-f96058b55495\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:08:12.529393727Z\",\n \"endTime\": \"2026-09-16T21:37:13.94502458Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m48s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-vtt65b86m6vb/cluster-candidate-4-21-gj8xr2 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c20920\u003e: \n failed to create HCP cluster cluster-candidate-4-21-gj8xr2: failed waiting for cluster=\"cluster-candidate-4-21-gj8xr2\" in resourcegroup=\"rg-candidate-4-21-vtt65b86m6vb\" to finish creating: GET https://rp.j3803776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9bcaf370-2958-4c5f-97cb-f96058b55495\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9bcaf370-2958-4c5f-97cb-f96058b55495\",\n \"name\": \"9bcaf370-2958-4c5f-97cb-f96058b55495\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T21:08:12.529393727Z\",\n \"endTime\": \"2026-09-16T21:37:13.94502458Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m48s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":9,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","f98a5615cab85c5989e6a0ac742f663ddff170e09cb4f716bf9a5978a279cf45"],"matched_occurrences":9,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100316086183399424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6359,"pr_state":"open","pr_sha":"e32a1089df44c9f8ef7f31132b66399727d8fb6d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T20:09:11Z","started_at":"2026-09-16T20:09:11Z","completed_at":"2026-09-16T20:09:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:05.002294272Z","timing_metadata_checked_at":"2026-09-18T15:40:05.002294272Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:05.005677884Z","region_metadata_checked_at":"2026-09-19T03:30:32.558170466Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"43c973738cbc6ea668aac5396631ebf8c6b1eecf0036904eeb97ee76d5269b5c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100316086183399424","occurred_at":"2026-09-16T20:09:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100314659356676096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6969,"pr_state":"open","pr_sha":"c4046b5e91e6786001b5754f90d8570f950ec5bd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T20:03:31Z","started_at":"2026-09-16T20:03:31Z","completed_at":"2026-09-16T22:04:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:01.544686103Z","timing_metadata_checked_at":"2026-09-18T15:41:01.544686103Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:01.547470913Z","region_metadata_checked_at":"2026-09-18T15:41:01.547470913Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"03a64504df4b7c809af4ae85c193ec865fb6dedb52e425b22f5ff3c86e5609ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100314659356676096","occurred_at":"2026-09-16T20:03:31Z","failed_at":"alert","signature_id":"d4858edb57d6296d8d8b3f66a13945cd04dcb772e32b66ac68791e2b7d64dc15","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T21:20:58Z\n Ended: 2026-09-16T21:49:56Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6676096-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-7d575bbb5-wsp69\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-7d575bbb5-wsp69 (namespace clusters-service) on cluster ci01-j6676096-svc has averaged 260.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T21:20:58Z\n Ended: 2026-09-16T21:38:56Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6676096-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-pvsvr\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-pvsvr (namespace arobit) on cluster ci01-j6676096-mgmt-1 has averaged 141.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-16T21:28:57Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6676096-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-7czlv\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-7czlv (namespace arobit) on cluster ci01-j6676096-mgmt-1 has averaged 186.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"66fab8d67f53098e691e746ed449be1ba5daa18eed2c47099bff9ba3472a0e82","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100314659356676096","occurred_at":"2026-09-16T20:03:31Z","failed_at":"e2e","signature_id":"ec03acc423d7d391cd0db46f80f6e282dd4d144bd87644aded93eda8c1bf31c4","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000edc320\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-d77hgbbbw6rf\" to finish creating: GET https://rp.j6676096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74e66356-25b3-4d6f-95af-f1292008b6fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74e66356-25b3-4d6f-95af-f1292008b6fd\",\n \"name\": \"74e66356-25b3-4d6f-95af-f1292008b6fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T20:47:22.69341748Z\",\n \"endTime\": \"2026-09-16T21:16:24.116227211Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 6m20s ago; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000edc320\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-d77hgbbbw6rf\" to finish creating: GET https://rp.j6676096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74e66356-25b3-4d6f-95af-f1292008b6fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74e66356-25b3-4d6f-95af-f1292008b6fd\",\n \"name\": \"74e66356-25b3-4d6f-95af-f1292008b6fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T20:47:22.69341748Z\",\n \"endTime\": \"2026-09-16T21:16:24.116227211Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 6m20s ago; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ed58550ec6b76673ce00588c8bebee5ecb6ea4c774ba1ceae5e31e8e80cd829","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7acfc1f51cd69ec58610f001bded1b14abc9ec4f8a1ddb43b9330166324b18b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100314659356676096","occurred_at":"2026-09-16T20:03:31Z","failed_at":"e2e","signature_id":"06e7cdde30cfbab4250217a65b741f57948c596ead351300b63debe1170a60a3","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001400078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-dqgbxbhrnvkj\" to finish deleting: GET https://rp.j6676096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38120612-3f62-4d83-b1c9-f450e308488b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38120612-3f62-4d83-b1c9-f450e308488b\",\n \"name\": \"38120612-3f62-4d83-b1c9-f450e308488b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T20:57:03.974907545Z\",\n \"endTime\": \"2026-09-16T21:21:04.12131988Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssb46njkre58n4p9mol1i33anuaqjtv still exists (deletion dispatched at 2026-09-16T20:57:05Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001400078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-dqgbxbhrnvkj\" to finish deleting: GET https://rp.j6676096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38120612-3f62-4d83-b1c9-f450e308488b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38120612-3f62-4d83-b1c9-f450e308488b\",\n \"name\": \"38120612-3f62-4d83-b1c9-f450e308488b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T20:57:03.974907545Z\",\n \"endTime\": \"2026-09-16T21:21:04.12131988Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssb46njkre58n4p9mol1i33anuaqjtv still exists (deletion dispatched at 2026-09-16T20:57:05Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","6ed58550ec6b76673ce00588c8bebee5ecb6ea4c774ba1ceae5e31e8e80cd829","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6945,"pr_state":"closed","pr_sha":"344450021ef1cd34320edd7790bea47a65b8b30e","final_merged_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T20:00:02Z","started_at":"2026-09-16T20:00:02Z","completed_at":"2026-09-16T21:59:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:49.19764329Z","timing_metadata_checked_at":"2026-09-18T15:40:49.19764329Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:49.201360703Z","region_metadata_checked_at":"2026-09-18T15:40:49.201360703Z"},"failed_at":["alert","e2e"],"failed_test_count":7,"occurrences":[{"row_id":"26c34a2a327eb0887034658a2e4218ed7173ea5f3b62f06ddd0caf3d2280b41b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","failed_at":"e2e","signature_id":"62f925cd8f107a148e561a08817704dd4da0dc08360a885c312c5776ffe6894d","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0008e6240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-x4ht6zckdvtk\" to finish deleting: GET https://rp.j6757888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/675795c0-1d48-4443-bf17-96b820fb9dab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/675795c0-1d48-4443-bf17-96b820fb9dab\",\n \"name\": \"675795c0-1d48-4443-bf17-96b820fb9dab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T20:56:31.8857491Z\",\n \"endTime\": \"2026-09-16T21:20:37.108128013Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssari2k27ic1890mj0nf8obfposs9em still exists (deletion dispatched at 2026-09-16T20:56:33Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0008e6240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-x4ht6zckdvtk\" to finish deleting: GET https://rp.j6757888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/675795c0-1d48-4443-bf17-96b820fb9dab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/675795c0-1d48-4443-bf17-96b820fb9dab\",\n \"name\": \"675795c0-1d48-4443-bf17-96b820fb9dab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T20:56:31.8857491Z\",\n \"endTime\": \"2026-09-16T21:20:37.108128013Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssari2k27ic1890mj0nf8obfposs9em still exists (deletion dispatched at 2026-09-16T20:56:33Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"867f2e94fd453baf2eb0f53eef73cb55d0acab81c98e808428d6f4743592e24c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","failed_at":"e2e","signature_id":"9cfc23c00b3b6ff4e037fcfa63ecc4ff6f6077b374d4b670e533df57b7d90b23","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0019540a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-downgrade-kttjnq\" in resourcegroup=\"rg-np-downgrade-kttjnq-75tphq26mtkt\" to finish deleting: GET https://rp.j6757888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/941782dd-efae-4093-940b-ca21585fc901\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/941782dd-efae-4093-940b-ca21585fc901\",\n \"name\": \"941782dd-efae-4093-940b-ca21585fc901\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T20:46:21.836683685Z\",\n \"endTime\": \"2026-09-16T21:10:27.160876667Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssarr47iva7e9en1e12mbat1fvbbvpq still exists (deletion dispatched at 2026-09-16T20:46:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0019540a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-downgrade-kttjnq\" in resourcegroup=\"rg-np-downgrade-kttjnq-75tphq26mtkt\" to finish deleting: GET https://rp.j6757888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/941782dd-efae-4093-940b-ca21585fc901\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/941782dd-efae-4093-940b-ca21585fc901\",\n \"name\": \"941782dd-efae-4093-940b-ca21585fc901\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T20:46:21.836683685Z\",\n \"endTime\": \"2026-09-16T21:10:27.160876667Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssarr47iva7e9en1e12mbat1fvbbvpq still exists (deletion dispatched at 2026-09-16T20:46:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"87649321ad723d9eb4c7e2e48dc7ef0e0a29372444007725a6bbcb9f6c55d585","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","failed_at":"alert","signature_id":"078c270d3b43c225b1670f600e86beb8fac31540b45fed0edb0e8f2b88023a38","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T21:02:59Z\n Ended: 2026-09-16T21:14:02Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6757888-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-jmb7z\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-jmb7z (namespace arobit) on cluster ci01-j6757888-mgmt-1 has averaged 165.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T21:02:59Z\n Ended: 2026-09-16T21:18:01Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6757888-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-f62zl\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-f62zl (namespace arobit) on cluster ci01-j6757888-mgmt-2 has averaged 179.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T21:02:59Z\n Ended: 2026-09-16T21:09:00Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6757888-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-844597f54-s6sj8\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-844597f54-s6sj8 (namespace clusters-service) on cluster ci01-j6757888-svc has averaged 249.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-16T21:02:59Z\n Ended: 2026-09-16T21:19:01Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6757888-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-89b689549-csthj\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-89b689549-csthj (namespace mgmt-agent) on cluster ci01-j6757888-mgmt-2 has averaged 199.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"92f3df24314cf8ad318b1d1654adaca457ebb19e6ac5c877e669f1cfb4ff8794","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","failed_at":"e2e","signature_id":"e259668ae1165ff12f8afd3ff5be66f96028094b8d861c7cbdff5a5c0c31f019","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001910318\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tfkgg85wrhjp\" to finish deleting: GET https://rp.j6757888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/40b6c313-a567-4259-bb1a-298290750f51\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/40b6c313-a567-4259-bb1a-298290750f51\",\n \"name\": \"40b6c313-a567-4259-bb1a-298290750f51\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T20:55:14.718059601Z\",\n \"endTime\": \"2026-09-16T21:19:17.098694741Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssarn8fr2snie5k70mjfefs0cg1miti still exists (deletion dispatched at 2026-09-16T20:55:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001910318\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tfkgg85wrhjp\" to finish deleting: GET https://rp.j6757888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/40b6c313-a567-4259-bb1a-298290750f51\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/40b6c313-a567-4259-bb1a-298290750f51\",\n \"name\": \"40b6c313-a567-4259-bb1a-298290750f51\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T20:55:14.718059601Z\",\n \"endTime\": \"2026-09-16T21:19:17.098694741Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssarn8fr2snie5k70mjfefs0cg1miti still exists (deletion dispatched at 2026-09-16T20:55:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9e2a2eae7ea1b6694866657178b3ff4e2b11773bde04aa98c7fbf55af6fd182c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","failed_at":"alert","signature_id":"7cedda9870b8f7ce3e9e32ac805564cf0bac96565ca3328c26875a1d70efde15","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T21:38:04Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j6757888-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-bcnth\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-bcnth (namespace arobit) on cluster ci01-j6757888-svc has averaged 127.2% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"9fbf2412a16ab8385bb1a4ef971d7ac1fde841604c738d5a677e897da9fda279","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","failed_at":"alert","signature_id":"d8457d1570ea90a26bcd98437f37670575aae28bffe0f986f9b004d2d9264040","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T20:41:10Z\n Ended: 2026-09-16T21:03:11Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6757888-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssarn0o77h8n6k7tcautgeeq1il2ldi-v20260901\", pod=\"router-65469f6bd6-n84vc\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssarn0o77h8n6k7tcautgeeq1il2ldi-v20260901/router-65469f6bd6-n84vc has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-16T20:42:08Z\n Ended: 2026-09-16T20:53:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6757888-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssari2k27ic1890mj0nf8obfposs9em-j6e9n3v1q9n1f1h\", pod=\"router-65684fdd94-xqlbm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssari2k27ic1890mj0nf8obfposs9em-j6e9n3v1q9n1f1h/router-65684fdd94-xqlbm has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"cceeb123eb88f0a2af212e2428a14b1afba6154596bce87a8108a77de3703a4d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","failed_at":"e2e","signature_id":"663948fad2f655a8f86778686b2675b46d2a717e3dc1afdecbbc520028b3d155","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:131]: nodepool creation errors: [failed to create NodePool np-update-nodes: failed waiting for nodepool=\"np-update-nodes\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-ckch4xg6nxh4\" to finish creating: GET https://rp.j6757888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e55eeddb-0ead-432e-a110-c696080abf84\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e55eeddb-0ead-432e-a110-c696080abf84\",\n \"name\": \"e55eeddb-0ead-432e-a110-c696080abf84\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T20:39:13.615216045Z\",\n \"endTime\": \"2026-09-16T20:58:16.977517435Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n \u003c[]error | len:1, cap:1\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:131]: nodepool creation errors: [failed to create NodePool np-update-nodes: failed waiting for nodepool=\"np-update-nodes\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-ckch4xg6nxh4\" to finish creating: GET https://rp.j6757888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e55eeddb-0ead-432e-a110-c696080abf84\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e55eeddb-0ead-432e-a110-c696080abf84\",\n \"name\": \"e55eeddb-0ead-432e-a110-c696080abf84\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T20:39:13.615216045Z\",\n \"endTime\": \"2026-09-16T20:58:16.977517435Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n \u003c[]error | len:1, cap:1\u003e: ...\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5","failure_pattern":"--------------------------------------------------------------------------------","search_query":"--------------------------------------------------------------------------------"}}],"failure_pattern_summary":{"signature_count":7,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5"],"matched_occurrences":7,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6904/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100310890917138432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6904,"pr_state":"open","pr_sha":"b5af817e18e018d2ea91c1c1ea1f25dedc56a009","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T19:48:32Z","started_at":"2026-09-16T19:48:32Z","completed_at":"2026-09-16T21:52:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:39.694475498Z","timing_metadata_checked_at":"2026-09-18T15:40:39.694475498Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:39.697551609Z","region_metadata_checked_at":"2026-09-18T15:40:39.697551609Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"56009e3d4e6007ca5a51480cf5b879f64a408b59aca11cf39e6f5f25c2b16891","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6904/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100310890917138432","occurred_at":"2026-09-16T19:48:32Z","failed_at":"alert","signature_id":"ba7f166d742007eec805103f097b0619e99ae0ecb99c10d11604e7456c1148b3","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T21:03:20Z\n Ended: 2026-09-16T21:20:20Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7138432-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-74fl9\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-74fl9 (namespace arobit) on cluster ci01-j7138432-mgmt-2 has averaged 149.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T21:07:21Z\n Ended: 2026-09-16T21:20:20Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7138432-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-58d785c466-8b8jc\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-58d785c466-8b8jc (namespace mgmt-agent) on cluster ci01-j7138432-mgmt-2 has averaged 165.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309676229922816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6894,"pr_state":"closed","pr_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","final_merged_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T19:43:43Z","started_at":"2026-09-16T19:43:43Z","completed_at":"2026-09-16T22:14:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:23.610206512Z","timing_metadata_checked_at":"2026-09-18T15:41:23.610206512Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:23.613674425Z","region_metadata_checked_at":"2026-09-18T15:41:23.613674425Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"ed106767857c0ee25b25a4c68b26474463081b9aa6a7027afecff22654f2a92a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309676229922816","occurred_at":"2026-09-16T19:43:43Z","failed_at":"e2e","signature_id":"a48d7255fe4f2e537c6260d9fc791f5067ce6859f69937f424d6c26f5bc86ca9","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f8a120\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-8prbhvsvlkq9\" to finish creating: GET https://rp.j9922816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b148abb4-8011-43a5-acf9-7d2f1f124b53\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b148abb4-8011-43a5-acf9-7d2f1f124b53\",\n \"name\": \"b148abb4-8011-43a5-acf9-7d2f1f124b53\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T20:56:50.517658972Z\",\n \"endTime\": \"2026-09-16T21:25:54.975318736Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f8a120\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-8prbhvsvlkq9\" to finish creating: GET https://rp.j9922816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b148abb4-8011-43a5-acf9-7d2f1f124b53\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b148abb4-8011-43a5-acf9-7d2f1f124b53\",\n \"name\": \"b148abb4-8011-43a5-acf9-7d2f1f124b53\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T20:56:50.517658972Z\",\n \"endTime\": \"2026-09-16T21:25:54.975318736Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5d59b4e4feb4fd290a5c1b398014539407c812b82baa8f933608971503b0bc0d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f4e2a62ddda1c67f3fb6e4decf73a131614f8fa6ded3447ef55c9e50740263c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309676229922816","occurred_at":"2026-09-16T19:43:43Z","failed_at":"alert","signature_id":"3a277ae7c258e0424f569baffe0a35ab11aec77aa3384cc607f2298340f9545f","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T21:58:47Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j9922816-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-8kkxq\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-8kkxq (namespace arobit) on cluster ci01-j9922816-svc has averaged 125.6% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"fca500a75e0760dd023ca2d7e80462b4f7b6e82709ee33b868b786dd46da3f34","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309676229922816","occurred_at":"2026-09-16T19:43:43Z","failed_at":"alert","signature_id":"d7f945bb6e41f7832c5eef4f84c2ae943bce6eff034300bf30a51c4a754e1cdc","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 5 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T21:18:46Z\n Ended: 2026-09-16T21:43:47Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9922816-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-6679f7776c-8zsx8\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-6679f7776c-8zsx8 (namespace aro-hcp) on cluster ci01-j9922816-svc has averaged 280.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T21:18:46Z\n Ended: 2026-09-16T21:43:47Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9922816-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-f87cdfc7-d2vpm\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-f87cdfc7-d2vpm (namespace clusters-service) on cluster ci01-j9922816-svc has averaged 555.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T21:18:46Z\n Ended: 2026-09-16T21:43:47Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9922816-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-f87cdfc7-kxvdm\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-f87cdfc7-kxvdm (namespace clusters-service) on cluster ci01-j9922816-svc has averaged 181.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-16T21:18:46Z\n Ended: 2026-09-16T21:43:47Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9922816-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-f87cdfc7-b8z5c\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-f87cdfc7-b8z5c (namespace clusters-service) on cluster ci01-j9922816-svc has averaged 221.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-16T21:22:47Z\n Ended: 2026-09-16T21:43:47Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9922816-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-8kkxq\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-8kkxq (namespace arobit) on cluster ci01-j9922816-svc has averaged 140.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","5d59b4e4feb4fd290a5c1b398014539407c812b82baa8f933608971503b0bc0d","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309343839719424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6403,"pr_state":"open","pr_sha":"39dc9f1a0c40de403279757f13edfa02afc80a0f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T19:42:24Z","started_at":"2026-09-16T19:42:24Z","completed_at":"2026-09-16T22:12:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:06.091865496Z","timing_metadata_checked_at":"2026-09-18T15:40:06.091865496Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:06.094867606Z","region_metadata_checked_at":"2026-09-18T15:40:06.094867606Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"90f75b7e4316f8d498cbc9018a810a3be012b5bd59fa563c27bf8a6ce228e209","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309343839719424","occurred_at":"2026-09-16T19:42:24Z","failed_at":"alert","signature_id":"efa1057a04dfc40b6e037e249cb0dc21b2c09a49fd06c904c78795387f5897e8","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 5 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T21:23:50Z\n Ended: 2026-09-16T21:31:49Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9719424-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-776c4f7db4-4kp88\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-776c4f7db4-4kp88 (namespace clusters-service) on cluster ci01-j9719424-svc has averaged 230.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T21:23:50Z\n Ended: 2026-09-16T21:31:49Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9719424-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-nj86k\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-nj86k (namespace arobit) on cluster ci01-j9719424-svc has averaged 155% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T21:23:50Z\n Ended: 2026-09-16T21:31:49Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9719424-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-bc48745b7-bx9p9\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-bc48745b7-bx9p9 (namespace aro-hcp) on cluster ci01-j9719424-svc has averaged 355.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-16T21:23:50Z\n Ended: 2026-09-16T21:39:50Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9719424-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-776c4f7db4-x52fn\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-776c4f7db4-x52fn (namespace clusters-service) on cluster ci01-j9719424-svc has averaged 236.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-16T21:23:50Z\n Ended: 2026-09-16T21:32:51Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9719424-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-776c4f7db4-8kcrd\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-776c4f7db4-8kcrd (namespace clusters-service) on cluster ci01-j9719424-svc has averaged 678% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"d9051ddf1c50966e73b9b4867345b6ad2578c96332058aa1af58d8bb6ed63f00","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309343839719424","occurred_at":"2026-09-16T19:42:24Z","failed_at":"e2e","signature_id":"af5ff2a9735a8e441c18465437503752dd1901dae01a546fbec14adb73b0c7e5","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-lzpm5g\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d707c0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-lzpm5g: failed waiting for cluster=\"cp-ystream-upgrade-4-22-lzpm5g\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-v5f9s54xdgwx\" to finish creating: GET https://rp.j9719424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cff0006a-d905-424f-9efd-d8db75f2fd05\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cff0006a-d905-424f-9efd-d8db75f2fd05\",\n \"name\": \"cff0006a-d905-424f-9efd-d8db75f2fd05\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T20:49:09.853286386Z\",\n \"endTime\": \"2026-09-16T21:18:11.008528958Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 13m4s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-lzpm5g\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d707c0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-lzpm5g: failed waiting for cluster=\"cp-ystream-upgrade-4-22-lzpm5g\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-v5f9s54xdgwx\" to finish creating: GET https://rp.j9719424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cff0006a-d905-424f-9efd-d8db75f2fd05\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cff0006a-d905-424f-9efd-d8db75f2fd05\",\n \"name\": \"cff0006a-d905-424f-9efd-d8db75f2fd05\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T20:49:09.853286386Z\",\n \"endTime\": \"2026-09-16T21:18:11.008528958Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 13m4s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c2687040eef152546e303fe3ce6da2f92a92db4c9a3989bfdcd2df869f68897f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","c2687040eef152546e303fe3ce6da2f92a92db4c9a3989bfdcd2df869f68897f"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309325879709696","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6928,"pr_state":"open","pr_sha":"dde72f49803d49335aafd35b01b79b6a92ac9d27","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T19:42:19Z","started_at":"2026-09-16T19:42:19Z","completed_at":"2026-09-16T21:45:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:42.497014584Z","timing_metadata_checked_at":"2026-09-18T15:40:42.497014584Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:42.500264596Z","region_metadata_checked_at":"2026-09-18T15:40:42.500264596Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"62f32536cc5c7280e6c3df25759ff80df04bb68a24ecf67daf1204edbb5ab837","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309325879709696","occurred_at":"2026-09-16T19:42:19Z","failed_at":"alert","signature_id":"c35a6da3cb62c984e490fa20921e5205ae8f9b5d5f473a81f2bc4d92f07256ce","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T21:08:14Z\n Ended: 2026-09-16T21:17:15Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9709696-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-657b9b568b-npr4g\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-657b9b568b-npr4g (namespace clusters-service) on cluster ci01-j9709696-svc has averaged 301.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T21:08:14Z\n Ended: 2026-09-16T21:17:16Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9709696-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-7d88d477f9-nx2ql\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-7d88d477f9-nx2ql (namespace mgmt-agent) on cluster ci01-j9709696-mgmt-2 has averaged 137.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T21:08:15Z\n Ended: 2026-09-16T21:19:13Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9709696-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-f2zcs\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-f2zcs (namespace arobit) on cluster ci01-j9709696-mgmt-1 has averaged 156% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309243423887360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4878,"pr_state":"closed","pr_sha":"8a73d0dda0fc968a36e83392cf2227e915acddc9","final_merged_sha":"244d0c16bcabe8284755cf089f7bb166cd38a435","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T19:42:00Z","started_at":"2026-09-16T19:42:00Z","completed_at":"2026-09-16T21:38:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:39:58.386919777Z","timing_metadata_checked_at":"2026-09-18T15:39:58.386919777Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:39:58.391309892Z","region_metadata_checked_at":"2026-09-18T15:39:58.391309892Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"ef6a1a01dcee40dd186a041dc595399aeace4ea69c7b7c221f5ce18a85b9cad6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309243423887360","occurred_at":"2026-09-16T19:42:00Z","failed_at":"alert","signature_id":"4e635a5a55fd9c70e99f4fc90a819fcbe842ed4f80ffca35e85d712a75adda03","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T20:58:29Z\n Ended: 2026-09-16T21:04:30Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3887360-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-n8wcv\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-n8wcv (namespace arobit) on cluster ci01-j3887360-mgmt-1 has averaged 210.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100299874200195072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6752,"pr_state":"closed","pr_sha":"d433c21668cfdc66f98262f9099c8086b4397248","final_merged_sha":"d433c21668cfdc66f98262f9099c8086b4397248","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T19:04:46Z","started_at":"2026-09-16T19:04:46Z","completed_at":"2026-09-16T21:02:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:20.025382239Z","timing_metadata_checked_at":"2026-09-18T15:40:20.025382239Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:20.02852925Z","region_metadata_checked_at":"2026-09-18T15:40:20.02852925Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"c8d645213e9f64ba33d7a46ec8828be7c1a50b8c26153680dc636aa7b199118d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100299874200195072","occurred_at":"2026-09-16T19:04:46Z","failed_at":"e2e","signature_id":"5264dd1fe40eb5bea2fb2d3c2020b90fcd8c767922fd709e8151a6988e36b5d7","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00108ec60\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-nzxcxjp96tgd\" to finish creating: GET https://rp.j0195072.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/68141ca8-41af-45e2-91cd-20fc07c3a848\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/68141ca8-41af-45e2-91cd-20fc07c3a848\",\n \"name\": \"68141ca8-41af-45e2-91cd-20fc07c3a848\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T19:52:50.957849824Z\",\n \"endTime\": \"2026-09-16T20:21:57.186960246Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m31s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00108ec60\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-nzxcxjp96tgd\" to finish creating: GET https://rp.j0195072.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/68141ca8-41af-45e2-91cd-20fc07c3a848\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/68141ca8-41af-45e2-91cd-20fc07c3a848\",\n \"name\": \"68141ca8-41af-45e2-91cd-20fc07c3a848\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T19:52:50.957849824Z\",\n \"endTime\": \"2026-09-16T20:21:57.186960246Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m31s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ca404f51abf125cd15be945bd1b0c641c6949d12024f1214d23811013841b898","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100299874200195072","occurred_at":"2026-09-16T19:04:46Z","failed_at":"alert","signature_id":"94b33034edb32ac378faab020acb7aa1cb586b97112ae6c5f8f64b5e0f39f1b3","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T20:25:13Z\n Ended: 2026-09-16T20:35:15Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0195072-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-7fb6485579-nqhtj\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-7fb6485579-nqhtj (namespace clusters-service) on cluster ci01-j0195072-svc has averaged 797.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T20:25:13Z\n Ended: 2026-09-16T20:35:15Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0195072-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-8zsgj\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-8zsgj (namespace arobit) on cluster ci01-j0195072-svc has averaged 278.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T20:26:13Z\n Ended: 2026-09-16T20:34:14Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0195072-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-kdkfn\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-kdkfn (namespace arobit) on cluster ci01-j0195072-mgmt-2 has averaged 148.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298749917007872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6966,"pr_state":"closed","pr_sha":"c93ff943c35ad2d21e3a98df5a27db691c9749a7","final_merged_sha":"c93ff943c35ad2d21e3a98df5a27db691c9749a7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T19:00:16Z","started_at":"2026-09-16T19:00:16Z","completed_at":"2026-09-16T20:54:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:00.295836707Z","timing_metadata_checked_at":"2026-09-18T15:41:00.295836707Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:00.298903018Z","region_metadata_checked_at":"2026-09-18T15:41:00.298903018Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"70ed40cbd900d749c00f79ece2db915648d0c657fa34986421f13b0a866674b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298749917007872","occurred_at":"2026-09-16T19:00:16Z","failed_at":"alert","signature_id":"4fe55ebf2e3687f10b470f9b50ef984b7f154b6ecd6235b377b7abf35f5fe025","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 8 time(s) (1 unknown, 7 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T19:54:25Z\n Ended: 2026-09-16T20:12:24Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7007872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssa6eeifmmv30pii215int1jih1qnml-g2b7m9d9z8p1d0i\", pod=\"router-577b4dc487-wqbd2\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssa6eeifmmv30pii215int1jih1qnml-g2b7m9d9z8p1d0i/router-577b4dc487-wqbd2 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"711d28f9cdbfa73f4b107a8ed249f27c8fedd45aa030debd0821ccb1404cd3c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298749917007872","occurred_at":"2026-09-16T19:00:16Z","failed_at":"alert","signature_id":"569b1e0aed53209c579e2ada482c82175262b769185ccf3a5bafd789625f23b0","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 7 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T20:16:48Z\n Ended: 2026-09-16T20:24:46Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7007872-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-qljsw\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-qljsw (namespace arobit) on cluster ci01-j7007872-mgmt-2 has averaged 177.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T20:16:48Z\n Ended: 2026-09-16T20:25:45Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7007872-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-77765956f4-qcl2v\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-77765956f4-qcl2v (namespace clusters-service) on cluster ci01-j7007872-svc has averaged 253.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T20:16:48Z\n Ended: 2026-09-16T20:32:47Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7007872-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-jbxwj\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-jbxwj (namespace arobit) on cluster ci01-j7007872-svc has averaged 267% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-16T20:16:48Z\n Ended: 2026-09-16T20:32:47Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7007872-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-77765956f4-9tcfp\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-77765956f4-9tcfp (namespace clusters-service) on cluster ci01-j7007872-svc has averaged 842.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-16T20:16:48Z\n Ended: 2026-09-16T20:33:49Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7007872-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-gjkhq\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-gjkhq (namespace arobit) on cluster ci01-j7007872-mgmt-2 has averaged 187.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Resolved\n Started: 2026-09-16T20:16:48Z\n Ended: 2026-09-16T20:32:47Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7007872-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-f8fcf57b5-bc6mp\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-f8fcf57b5-bc6mp (namespace aro-hcp) on cluster ci01-j7007872-svc has averaged 408.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 7:\n State: Resolved\n Started: 2026-09-16T20:16:48Z\n Ended: 2026-09-16T20:30:46Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7007872-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-74f949fc76-tvpjx\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-74f949fc76-tvpjx (namespace mgmt-agent) on cluster ci01-j7007872-mgmt-2 has averaged 191% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298527354654720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6976,"pr_state":"open","pr_sha":"3f37770fba22a55b4336e3a85cbf275bf37c8279","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T18:59:25Z","started_at":"2026-09-16T18:59:25Z","completed_at":"2026-09-16T21:02:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:05.033954567Z","timing_metadata_checked_at":"2026-09-18T15:41:05.033954567Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:05.037220379Z","region_metadata_checked_at":"2026-09-18T15:41:05.037220379Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"2f284be9477c48cf174659ad5934e41ad57bccb858989ed3ba73fefc8f13411b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298527354654720","occurred_at":"2026-09-16T18:59:25Z","failed_at":"alert","signature_id":"2515546ca4a2ed9cc66209f7f5a861f329da8544f35e1712e6bef48626b727f0","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T20:17:53Z\n Ended: 2026-09-16T20:29:53Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4654720-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-5f6fb8bd86-rq9t9\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-5f6fb8bd86-rq9t9 (namespace mgmt-agent) on cluster ci01-j4654720-mgmt-1 has averaged 182% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T20:17:53Z\n Ended: 2026-09-16T20:36:52Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4654720-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-59db946469-cmfdh\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-59db946469-cmfdh (namespace clusters-service) on cluster ci01-j4654720-svc has averaged 244% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T20:17:53Z\n Ended: 2026-09-16T20:29:53Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4654720-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-wwfl9\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-wwfl9 (namespace arobit) on cluster ci01-j4654720-mgmt-1 has averaged 232.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"b17f36fee7834c91510c6fdae8b4bb6710dab9aeebcafd9e98d5569442d49444","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298527354654720","occurred_at":"2026-09-16T18:59:25Z","failed_at":"e2e","signature_id":"bf7ebbd181ec7aa5127a5794e1778bd8570a9bdf75c05ec0454a8c04349e2cae","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:738]: failed to create nodepool npdg-422-420\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a2c540\u003e: \n failed to create NodePool npdg-422-420: failed waiting for nodepool=\"npdg-422-420\" for cluster \"np-dg-minor-86c2sh\" in resourcegroup=\"rg-np-dg-minor-86c2sh-42qzgk6zdhqx\" to finish creating: GET https://rp.j4654720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b168b436-e53c-4032-86e5-822b4fbf8df0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b168b436-e53c-4032-86e5-822b4fbf8df0\",\n \"name\": \"b168b436-e53c-4032-86e5-822b4fbf8df0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T19:54:19.817293417Z\",\n \"endTime\": \"2026-09-16T19:54:53.786912449Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:738]: failed to create nodepool npdg-422-420\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a2c540\u003e: \n failed to create NodePool npdg-422-420: failed waiting for nodepool=\"npdg-422-420\" for cluster \"np-dg-minor-86c2sh\" in resourcegroup=\"rg-np-dg-minor-86c2sh-42qzgk6zdhqx\" to finish creating: GET https://rp.j4654720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b168b436-e53c-4032-86e5-822b4fbf8df0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b168b436-e53c-4032-86e5-822b4fbf8df0\",\n \"name\": \"b168b436-e53c-4032-86e5-822b4fbf8df0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T19:54:19.817293417Z\",\n \"endTime\": \"2026-09-16T19:54:53.786912449Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298495381475328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6977,"pr_state":"closed","pr_sha":"557c04379e033b9de0bd85904b49103f2cdef1b1","final_merged_sha":"557c04379e033b9de0bd85904b49103f2cdef1b1","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T18:59:17Z","started_at":"2026-09-16T18:59:17Z","completed_at":"2026-09-16T20:49:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:06.222806248Z","timing_metadata_checked_at":"2026-09-18T15:41:06.222806248Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:06.226258861Z","region_metadata_checked_at":"2026-09-18T15:41:06.226258861Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"06dd38bd3dfe4aa46c5faed25f9a4c30e92e0e10eae121bad2d4d92d3004b3cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298495381475328","occurred_at":"2026-09-16T18:59:17Z","failed_at":"alert","signature_id":"ed3bf953568238e27f1b875c5f5f12b7c39936c7b777f8037f83dc0b1f8c2feb","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 8 time(s) (2 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T19:51:28Z\n Ended: 2026-09-16T20:12:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1475328-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssa495v0fcuam12oc6m4b88gdkl9b05-l2h4f1u2r7a1i0b\", pod=\"router-85855657b9-6nz7v\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssa495v0fcuam12oc6m4b88gdkl9b05-l2h4f1u2r7a1i0b/router-85855657b9-6nz7v has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-16T19:51:28Z\n Ended: 2026-09-16T20:03:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1475328-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssa4occ93fjcinhaca51fjlpiva3rpe-private-ingress\", pod=\"router-68ff4878d4-n82h2\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssa4occ93fjcinhaca51fjlpiva3rpe-private-ingress/router-68ff4878d4-n82h2 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"7cb6050551e4f4f9c21524b36dd24bbdb095d0963ac19b0db7b83225a28ba1af","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298495381475328","occurred_at":"2026-09-16T18:59:17Z","failed_at":"e2e","signature_id":"6b84028a0066d5af15d34aa2f3390d5b3b8c5439b097d46537c60eee00e895c1","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.000s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0007c5690\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.000s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0007c5690\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}},{"row_id":"beaaf4a8f2b2537d0ea7b3eacf912571ffb514886466cb272c413ac7e9a4e00b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298495381475328","occurred_at":"2026-09-16T18:59:17Z","failed_at":"alert","signature_id":"b7bbff7c4bc8e38db57964016b3e5dd22cbf69711555869ce39fed0d057ef5fa","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T20:11:42Z\n Ended: 2026-09-16T20:23:44Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j1475328-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-wphcl\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-wphcl (namespace arobit) on cluster ci01-j1475328-mgmt-2 has averaged 210.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T20:11:42Z\n Ended: 2026-09-16T20:24:43Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j1475328-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-5cbb7b949c-mbxpl\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-5cbb7b949c-mbxpl (namespace mgmt-agent) on cluster ci01-j1475328-mgmt-2 has averaged 198.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T20:12:45Z\n Ended: 2026-09-16T20:29:43Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j1475328-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-xrf59\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-xrf59 (namespace arobit) on cluster ci01-j1475328-mgmt-1 has averaged 140.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-16T20:14:43Z\n Ended: 2026-09-16T20:26:44Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j1475328-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-nzczq\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-nzczq (namespace arobit) on cluster ci01-j1475328-mgmt-2 has averaged 139.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100294017282478080","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6980,"pr_state":"closed","pr_sha":"4c6ef86c4410d1e48dd7653c177a4370cf38a246","final_merged_sha":"4c6ef86c4410d1e48dd7653c177a4370cf38a246","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T18:41:29Z","started_at":"2026-09-16T18:41:29Z","completed_at":"2026-09-16T20:38:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:07.635322835Z","timing_metadata_checked_at":"2026-09-18T15:41:07.635322835Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:07.639303249Z","region_metadata_checked_at":"2026-09-18T15:41:07.639303249Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"96a57ec6f7aeeaef63157101efa47ca2587c95763ec10ca8367840bdec9ace23","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100294017282478080","occurred_at":"2026-09-16T18:41:29Z","failed_at":"alert","signature_id":"d6b769c42dfc7af863ab56e421478bc2130ed7b8030470653f093468c2ae9445","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 7 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T20:04:07Z\n Ended: 2026-09-16T20:16:07Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2478080-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-7bb74f5747-292f6\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-7bb74f5747-292f6 (namespace mgmt-agent) on cluster ci01-j2478080-mgmt-1 has averaged 172.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T20:04:07Z\n Ended: 2026-09-16T20:26:13Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2478080-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-56db9df8f4-5fz7b\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-56db9df8f4-5fz7b (namespace aro-hcp) on cluster ci01-j2478080-svc has averaged 321.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T20:04:07Z\n Ended: 2026-09-16T20:14:08Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2478080-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-d5fts\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-d5fts (namespace arobit) on cluster ci01-j2478080-mgmt-2 has averaged 154.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-16T20:04:08Z\n Ended: 2026-09-16T20:26:12Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2478080-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-fd28t\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-fd28t (namespace arobit) on cluster ci01-j2478080-svc has averaged 203.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-16T20:04:08Z\n Ended: 2026-09-16T20:26:13Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2478080-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-5d449f989c-sx24g\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-5d449f989c-sx24g (namespace clusters-service) on cluster ci01-j2478080-svc has averaged 663.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Resolved\n Started: 2026-09-16T20:04:08Z\n Ended: 2026-09-16T20:14:09Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2478080-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-6fb29\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-6fb29 (namespace arobit) on cluster ci01-j2478080-mgmt-1 has averaged 148.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 7:\n State: Resolved\n Started: 2026-09-16T20:04:08Z\n Ended: 2026-09-16T20:16:07Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2478080-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-7bb74f5747-7zxzr\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-7bb74f5747-7zxzr (namespace mgmt-agent) on cluster ci01-j2478080-mgmt-2 has averaged 181.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"eda6acd3c210f993ac6fd94df55d25b4cd5cb0842869b6fadae3791632ba1694","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100294017282478080","occurred_at":"2026-09-16T18:41:29Z","failed_at":"e2e","signature_id":"5b5a6e4f2745283c541cfa867e89616d0e7a33b663b83015979c872710cb256e","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00091cf60\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-cdklhkjpj64p\" to finish creating: GET https://rp.j2478080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/da51fff3-7c72-4b61-92a1-b6c1ea3a06a4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/da51fff3-7c72-4b61-92a1-b6c1ea3a06a4\",\n \"name\": \"da51fff3-7c72-4b61-92a1-b6c1ea3a06a4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T19:31:13.552925362Z\",\n \"endTime\": \"2026-09-16T20:00:19.71689726Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m45s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00091cf60\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-cdklhkjpj64p\" to finish creating: GET https://rp.j2478080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/da51fff3-7c72-4b61-92a1-b6c1ea3a06a4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/da51fff3-7c72-4b61-92a1-b6c1ea3a06a4\",\n \"name\": \"da51fff3-7c72-4b61-92a1-b6c1ea3a06a4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T19:31:13.552925362Z\",\n \"endTime\": \"2026-09-16T20:00:19.71689726Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m45s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f8663542543e0821255ff93a9707bb220197151be5ea9696eec4793abfa727f0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100294017282478080","occurred_at":"2026-09-16T18:41:29Z","failed_at":"e2e","signature_id":"30b7eed5c08665fa2cef128c104b6bc116a828b17cff3a0328ac6b1a7aaf5b24","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.006s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00199ac80\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:35223-\u003e172.30.0.10:53: read: connection refused\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.006s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00199ac80\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:35223-\u003e172.30.0.10:53: read: connection refused\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100293442402783232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6805,"pr_state":"closed","pr_sha":"b3bec0e87c8b5d73d493df4173e57e3c3590d2d7","final_merged_sha":"b3bec0e87c8b5d73d493df4173e57e3c3590d2d7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T18:39:12Z","started_at":"2026-09-16T18:39:12Z","completed_at":"2026-09-16T20:37:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:24.347143287Z","timing_metadata_checked_at":"2026-09-18T15:40:24.347143287Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:24.350240899Z","region_metadata_checked_at":"2026-09-18T15:40:24.350240899Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"cfcc3f90e292971aef7800db35afc280ac75f35bdaaf25714dd9d962d5460be0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100293442402783232","occurred_at":"2026-09-16T18:39:12Z","failed_at":"alert","signature_id":"6d07e5dea3d8417306b6aad1d07607ec094c290a529075bc5ae0f6c507ae8dc7","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T19:58:36Z\n Ended: 2026-09-16T20:12:38Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2783232-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-99f86f48f-z26v6\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-99f86f48f-z26v6 (namespace mgmt-agent) on cluster ci01-j2783232-mgmt-2 has averaged 219.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-16T19:58:36Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2783232-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-76d699cfb8-xkqc4\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-76d699cfb8-xkqc4 (namespace clusters-service) on cluster ci01-j2783232-svc has averaged 915.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T19:58:36Z\n Ended: 2026-09-16T20:12:38Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2783232-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-gjlth\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-gjlth (namespace arobit) on cluster ci01-j2783232-mgmt-2 has averaged 196.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-16T19:59:38Z\n Ended: 2026-09-16T20:07:36Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2783232-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-fm5qj\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-fm5qj (namespace arobit) on cluster ci01-j2783232-mgmt-1 has averaged 137.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"f2d30f3db4605a654f09a6300fea1ec08e96beffa826c9ba32cb39fd42339b4d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100293442402783232","occurred_at":"2026-09-16T18:39:12Z","failed_at":"e2e","signature_id":"3d2d19c49b473c9b2b5c1bd828e9238e2a0bdd4c4f8d89fe2be19a6ea937a4b7","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000db98e0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-5kbxx5fn4nnf\" to finish creating: GET https://rp.j2783232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ff402ee6-ae9a-437a-b814-5ad8ed11cf52\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ff402ee6-ae9a-437a-b814-5ad8ed11cf52\",\n \"name\": \"ff402ee6-ae9a-437a-b814-5ad8ed11cf52\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T19:26:24.241613754Z\",\n \"endTime\": \"2026-09-16T19:55:27.054305259Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000db98e0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-5kbxx5fn4nnf\" to finish creating: GET https://rp.j2783232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ff402ee6-ae9a-437a-b814-5ad8ed11cf52\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ff402ee6-ae9a-437a-b814-5ad8ed11cf52\",\n \"name\": \"ff402ee6-ae9a-437a-b814-5ad8ed11cf52\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T19:26:24.241613754Z\",\n \"endTime\": \"2026-09-16T19:55:27.054305259Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100285588732121088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"b368522ea717841a0bf803ebbe3e8dc7455b6e5e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T18:08:00Z","started_at":"2026-09-16T18:08:00Z","completed_at":"2026-09-16T20:06:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:05.47386747Z","timing_metadata_checked_at":"2026-09-18T15:40:05.47386747Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:05.476961881Z","region_metadata_checked_at":"2026-09-18T15:40:05.476961881Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"5e2130ef288e43f8897baa719aa82ba28c64b25d75234eb645a25355e76dae13","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100285588732121088","occurred_at":"2026-09-16T18:08:00Z","failed_at":"alert","signature_id":"3165e7829bfab81efada837f8a5a846a713826a1a775685756db96ad2a2e7a0b","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 5 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T19:21:11Z\n Ended: 2026-09-16T19:31:08Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2121088-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-rxj44\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-rxj44 (namespace arobit) on cluster ci01-j2121088-svc has averaged 355.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T19:21:11Z\n Ended: 2026-09-16T19:31:08Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2121088-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-6b56bfd454-hdslm\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-6b56bfd454-hdslm (namespace aro-hcp) on cluster ci01-j2121088-svc has averaged 552.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T19:21:11Z\n Ended: 2026-09-16T19:31:08Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2121088-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-747fb55f57-d9rdv\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-747fb55f57-d9rdv (namespace clusters-service) on cluster ci01-j2121088-svc has averaged 960.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-16T19:23:13Z\n Ended: 2026-09-16T19:33:11Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2121088-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-67b9d68855-jzclk\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-67b9d68855-jzclk (namespace mgmt-agent) on cluster ci01-j2121088-mgmt-1 has averaged 150.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-16T19:23:13Z\n Ended: 2026-09-16T19:38:12Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2121088-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-8lsww\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-8lsww (namespace arobit) on cluster ci01-j2121088-mgmt-1 has averaged 158.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"7bdbd6a1bca21be31d9492359e045c190592bcc6040d2b1aa5af13933e7c833b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100285588732121088","occurred_at":"2026-09-16T18:08:00Z","failed_at":"e2e","signature_id":"7f3c6a5d4502814ef937d1139cc649fca2dcf5ac7887cdb4d24e35c02e2550c3","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:480]: Timed out after 34.100s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc001341f60\u003e: \n permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /endpoints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /events: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /persistentvolumeclaims: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for get /persistentvolumes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /serviceaccounts: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /resourcequotas: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /limitranges: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /replicationcontrollers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for get /componentstatuses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for get /pods/log: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apps/deployments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apps/daemonsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apps/statefulsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apps/replicasets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list batch/jobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list batch/cronjobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list networking.k8s.io/networkpolicies: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list networking.k8s.io/ingresses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list rbac.authorization.k8s.io/clusterroles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list rbac.authorization.k8s.io/clusterrolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list rbac.authorization.k8s.io/roles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list rbac.authorization.k8s.io/rolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list storage.k8s.io/storageclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:480]: Timed out after 34.100s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc001341f60\u003e: \n permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /endpoints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /events: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /persistentvolumeclaims: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for get /persistentvolumes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /serviceaccounts: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /resourcequotas: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /limitranges: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /replicationcontrollers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for get /componentstatuses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for get /pods/log: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apps/deployments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apps/daemonsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apps/statefulsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apps/replicasets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list batch/jobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list batch/cronjobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list networking.k8s.io/networkpolicies: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list networking.k8s.io/ingresses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list rbac.authorization.k8s.io/clusterroles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list rbac.authorization.k8s.io/clusterrolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list rbac.authorization.k8s.io/roles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list rbac.authorization.k8s.io/rolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list storage.k8s.io/storageclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"50012cc84060fc59db5e840f9933bf1a542baf6598061d9d1e54da53aa955055","failure_pattern":"permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews","search_query":"permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews"}},{"row_id":"8307304ac8f511eba9d6d2caba3474202d8cd19b2e1a6a831be4ba1928d1b3a4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100285588732121088","occurred_at":"2026-09-16T18:08:00Z","failed_at":"alert","signature_id":"040c54783eeb571a32188be480f0e0617f1911469f8d08b133b147d373df8efb","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T19:40:11Z\n Ended: 2026-09-16T19:46:13Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j2121088-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-rxj44\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-rxj44 (namespace arobit) on cluster ci01-j2121088-svc has averaged 126.6% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","50012cc84060fc59db5e840f9933bf1a542baf6598061d9d1e54da53aa955055","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100283069305982976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6965,"pr_state":"closed","pr_sha":"ac0717465df81eeeb7d8446cc4182a603bb8004f","final_merged_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T17:57:59Z","started_at":"2026-09-16T17:57:59Z","completed_at":"2026-09-16T18:28:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:59.771920723Z","timing_metadata_checked_at":"2026-09-18T15:40:59.771920723Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:59.775710036Z","region_metadata_checked_at":"2026-09-18T15:40:59.775710036Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"751a5b82c5b59280a5c20494dffa4219de3954ec069d54763e9cdaba03811995","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100283069305982976","occurred_at":"2026-09-16T17:57:59Z","failed_at":"provision","signature_id":"2a5637582a78b2337c484f5a7a9ff7f9d84aa702c7b086e89182f1549eb13d57","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Monitoring/regional/monitoring","test_suite":"step graph","failure_text":"time=2026-09-16T18:22:02.494Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring description=\"Step monitoring\\n Kind: ARMStack\\n Template: templates/monitoring.bicep\\n Parameters: configurations/monitoring.tmpl.bicepparam\"\ntime=2026-09-16T18:22:51.675Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring err=\"failed to run ARM step: failed to wait for deployment stack at resource group scope: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/canadacentral/deploymentStackOperationStatus/1df7c411-d9ac-443d-ac5d-b95681f8419b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentStackDeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/canadacentral/deploymentStackOperationStatus/1df7c411-d9ac-443d-ac5d-b95681f8419b\\\",\\n \\\"name\\\": \\\"1df7c411-d9ac-443d-ac5d-b95681f8419b\\\",\\n \\\"status\\\": \\\"failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentStackDeploymentFailed\\\",\\n \\\"message\\\": \\\"One or more resources could not be deployed. Correlation id: 'd478c792-be07-45be-a743-97b4e94abf15'.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5982976/providers/Microsoft.Resources/deployments/nbyngfosfawbsx5vtxb2uo6lmfim2rbaopp3emm2zwlmm-260916186gla8\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"Property name: Rules[0].Expression, Attempted value: null, Error: 1:237: parse error: unknown escape sequence U+002E '.' Activity ID: 95e8cbf4-5734-47ec-859e-d6ed81e0056f.\\\"\\n }\\n ]\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e405dc3ba5eaa76b269136ece3c7ec09bc9d5c543da581d8b5e14a4da0eb2410","failure_pattern":"ERROR CODE: DeploymentStackDeploymentFailed","search_query":"ERROR CODE: DeploymentStackDeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e405dc3ba5eaa76b269136ece3c7ec09bc9d5c543da581d8b5e14a4da0eb2410"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100283006785687552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6952,"pr_state":"closed","pr_sha":"a4de05dadb773673a7d1a63ee2677dbb182bda10","final_merged_sha":"a4de05dadb773673a7d1a63ee2677dbb182bda10","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-16T17:57:44Z","started_at":"2026-09-16T17:57:44Z","completed_at":"2026-09-16T19:57:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:57.40783382Z","timing_metadata_checked_at":"2026-09-18T15:40:57.40783382Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:57.410948731Z","region_metadata_checked_at":"2026-09-18T15:40:57.410948731Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6923/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100282979875033088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6923,"pr_state":"closed","pr_sha":"9e6650394c80bca242363572272841073766ea22","final_merged_sha":"9e6650394c80bca242363572272841073766ea22","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-16T17:57:38Z","started_at":"2026-09-16T17:57:38Z","completed_at":"2026-09-16T19:52:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:41.57047135Z","timing_metadata_checked_at":"2026-09-18T15:40:41.57047135Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:41.573767461Z","region_metadata_checked_at":"2026-09-18T15:40:41.573767461Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100280971763912704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6945,"pr_state":"closed","pr_sha":"344450021ef1cd34320edd7790bea47a65b8b30e","final_merged_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T17:49:39Z","started_at":"2026-09-16T17:49:39Z","completed_at":"2026-09-16T19:57:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:49.030290888Z","timing_metadata_checked_at":"2026-09-18T15:40:49.030290888Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:49.035615507Z","region_metadata_checked_at":"2026-09-18T15:40:49.035615507Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"191ab1a405d7eef56e134bb03684ebc5681ec179f442628e3de66a7a5770eca7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100280971763912704","occurred_at":"2026-09-16T17:49:39Z","failed_at":"e2e","signature_id":"e4e6c14d747d24bff47997187743a4306c16de10ca6e0660e678007cc3c9c76a","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0xc002420258\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-hgxdxgb7d5pf\" to finish creating: GET https://rp.j3912704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc0b2672-7030-4135-9a4f-3ac73ff9d19d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc0b2672-7030-4135-9a4f-3ac73ff9d19d\",\n \"name\": \"bc0b2672-7030-4135-9a4f-3ac73ff9d19d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T18:52:16.312207682Z\",\n \"endTime\": \"2026-09-16T18:52:49.366400193Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0xc002420258\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-hgxdxgb7d5pf\" to finish creating: GET https://rp.j3912704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc0b2672-7030-4135-9a4f-3ac73ff9d19d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc0b2672-7030-4135-9a4f-3ac73ff9d19d\",\n \"name\": \"bc0b2672-7030-4135-9a4f-3ac73ff9d19d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T18:52:16.312207682Z\",\n \"endTime\": \"2026-09-16T18:52:49.366400193Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"587565af465ff16da21d2a85e8baf74a7ec302383bf07c48a0fb1b2ef2d2a622","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100280971763912704","occurred_at":"2026-09-16T17:49:39Z","failed_at":"alert","signature_id":"5241ddfc74d49c5ecad2616742ded97b4a5f9bf5fbeee0f248252a9d33303902","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T19:19:40Z\n Ended: 2026-09-16T19:27:40Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3912704-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-5cd45c48f-qb4qc\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-5cd45c48f-qb4qc (namespace clusters-service) on cluster ci01-j3912704-svc has averaged 808.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T19:19:40Z\n Ended: 2026-09-16T19:27:40Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3912704-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-6f7fcd76b-7dnrd\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-6f7fcd76b-7dnrd (namespace aro-hcp) on cluster ci01-j3912704-svc has averaged 423.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T19:19:41Z\n Ended: 2026-09-16T19:27:40Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3912704-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-mv6rx\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-mv6rx (namespace arobit) on cluster ci01-j3912704-svc has averaged 236.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-16T19:19:41Z\n Ended: 2026-09-16T19:26:39Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3912704-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-5cd45c48f-x4czc\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-5cd45c48f-x4czc (namespace clusters-service) on cluster ci01-j3912704-svc has averaged 243.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100273649465430016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6894,"pr_state":"closed","pr_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","final_merged_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T17:20:33Z","started_at":"2026-09-16T17:20:33Z","completed_at":"2026-09-16T19:40:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:23.474856722Z","timing_metadata_checked_at":"2026-09-18T15:41:23.474856722Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:23.478065934Z","region_metadata_checked_at":"2026-09-18T15:41:23.478065934Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"34cbcb96ab17f8ef527bb00f6fd956cf36da670ae7ec234ef1ca9dd00ff4c37c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100273649465430016","occurred_at":"2026-09-16T17:20:33Z","failed_at":"e2e","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:170]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred. Got:\n \u003cnil\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:170]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred. Got:\n \u003cnil\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"406d47e638f407d24104655febf18e753de33546f725682f49558d9bc4ae2365","failure_pattern":"private ingress should not be reachable from outside the VNet, but connection succeeded","search_query":"private ingress should not be reachable from outside the VNet, but connection succeeded"}},{"row_id":"8d8e903974552faae39b72c6004932eef0096f70d39ac38792d95bd2529c8fbd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100273649465430016","occurred_at":"2026-09-16T17:20:33Z","failed_at":"alert","signature_id":"de52e1f3238a7b10dc025230a0bc30a7b49fe207de3da3e23620f54404272671","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T19:14:12Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j5430016-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-tcbmj\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-tcbmj (namespace arobit) on cluster ci01-j5430016-svc has averaged 124.5% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"92a0296176c0b89dc87d77d363779ac179261ffb21b7ed378db7a4649b1be825","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100273649465430016","occurred_at":"2026-09-16T17:20:33Z","failed_at":"alert","signature_id":"2aaecc7a8aa85a870eddb45d3a454574d39a17a2e1971a571ef303ebfa1b5dfa","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 14 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T18:35:14Z\n Ended: 2026-09-16T19:14:15Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5430016-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-7767bb5cf4-9nfxx\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-7767bb5cf4-9nfxx (namespace clusters-service) on cluster ci01-j5430016-svc has averaged 175.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T18:35:15Z\n Ended: 2026-09-16T18:57:15Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5430016-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-7767bb5cf4-9smcf\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-7767bb5cf4-9smcf (namespace clusters-service) on cluster ci01-j5430016-svc has averaged 175.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T18:35:15Z\n Ended: 2026-09-16T19:21:18Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5430016-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-8598b596fb-8scn6\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-8598b596fb-8scn6 (namespace aro-hcp) on cluster ci01-j5430016-svc has averaged 335.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-16T18:35:15Z\n Ended: 2026-09-16T19:18:19Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5430016-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-tcbmj\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-tcbmj (namespace arobit) on cluster ci01-j5430016-svc has averaged 207.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-16T18:35:15Z\n Ended: 2026-09-16T19:29:15Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5430016-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-7767bb5cf4-mz556\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-7767bb5cf4-mz556 (namespace clusters-service) on cluster ci01-j5430016-svc has averaged 547.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Resolved\n Started: 2026-09-16T18:46:14Z\n Ended: 2026-09-16T18:59:15Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5430016-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-xwc94\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-xwc94 (namespace arobit) on cluster ci01-j5430016-mgmt-1 has averaged 142% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 7:\n State: Resolved\n Started: 2026-09-16T18:46:14Z\n Ended: 2026-09-16T18:59:15Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5430016-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-8fc8c5486-9xpzk\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-8fc8c5486-9xpzk (namespace mgmt-agent) on cluster ci01-j5430016-mgmt-1 has averaged 189.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 8:\n State: Resolved\n Started: 2026-09-16T18:46:14Z\n Ended: 2026-09-16T19:02:21Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5430016-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-tmx55\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-tmx55 (namespace arobit) on cluster ci01-j5430016-mgmt-1 has averaged 133.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 9:\n State: Resolved\n Started: 2026-09-16T18:47:15Z\n Ended: 2026-09-16T18:54:16Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5430016-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-g6qvv\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-g6qvv (namespace arobit) on cluster ci01-j5430016-mgmt-2 has averaged 146.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 10:\n State: Resolved\n Started: 2026-09-16T18:47:15Z\n Ended: 2026-09-16T19:02:21Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5430016-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-whfrb\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-whfrb (namespace arobit) on cluster ci01-j5430016-mgmt-2 has averaged 132.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 11:\n State: Resolved\n Started: 2026-09-16T18:47:15Z\n Ended: 2026-09-16T19:04:14Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5430016-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-lzct4\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-lzct4 (namespace arobit) on cluster ci01-j5430016-mgmt-1 has averaged 137.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 12:\n State: Resolved\n Started: 2026-09-16T18:48:15Z\n Ended: 2026-09-16T18:54:16Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5430016-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-8fc8c5486-klh5t\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-8fc8c5486-klh5t (namespace mgmt-agent) on cluster ci01-j5430016-mgmt-2 has averaged 159.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 13:\n State: Resolved\n Started: 2026-09-16T18:52:14Z\n Ended: 2026-09-16T19:03:13Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5430016-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-kqxnt\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-kqxnt (namespace arobit) on cluster ci01-j5430016-mgmt-1 has averaged 136.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 14:\n State: Resolved\n Started: 2026-09-16T18:54:16Z\n Ended: 2026-09-16T19:03:13Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5430016-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-ncwrg\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-ncwrg (namespace arobit) on cluster ci01-j5430016-mgmt-1 has averaged 129.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","406d47e638f407d24104655febf18e753de33546f725682f49558d9bc4ae2365","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100270038647836672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6820,"pr_state":"open","pr_sha":"c99d95a578f89e3e4c3bfc476a031bb4b59056f4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T17:06:12Z","started_at":"2026-09-16T17:06:12Z","completed_at":"2026-09-16T18:07:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:25.454257071Z","timing_metadata_checked_at":"2026-09-18T15:40:25.454257071Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:25.457409282Z","region_metadata_checked_at":"2026-09-18T15:40:25.457409282Z"},"failed_at":["alert","provision"],"failed_test_count":4,"occurrences":[{"row_id":"929ea11d6dc25da82505d7af3307ad18d7404136ca38013d64a0ecb7922e8803","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100270038647836672","occurred_at":"2026-09-16T17:06:12Z","failed_at":"alert","signature_id":"ce202b7ce3d72dcf4b71c51db3c021d84810e4ab274c64dae9e86007cff0fec7","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T17:45:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7836672-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-59s8j\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-59s8j has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-16T17:45:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7836672-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-9zlkb\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-9zlkb has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-16T17:45:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7836672-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-t8hdl\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-t8hdl has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-16T17:45:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7836672-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-49tpv\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-49tpv has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"ccf70148625b497cb39ec48ace65d881ac97db5ee7b0e806cfaa234b21618107","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100270038647836672","occurred_at":"2026-09-16T17:06:12Z","failed_at":"alert","signature_id":"c1988f523c9b1951a937f22e869077937abcdb055a057f4fd9deb538ebdeeab8","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T17:55:54Z\n Severity: Sev3\n Labels: alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j7836672-svc\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"warning\"\n Description: The Arobit forwarder metrics endpoint on cluster ci01-j7836672-svc has been unreachable for 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired"}},{"row_id":"e3cf45b1ad68700369fc07902575b29d241d5ea68283cb12627a8c70b73a73e1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100270038647836672","occurred_at":"2026-09-16T17:06:12Z","failed_at":"provision","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","test_suite":"step graph","failure_text":"time=2026-09-16T17:32:16.355Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-16T17:32:17.069Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-16T17:32:18.139Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=2220b5a771141f6ca7f14ed4462ae3b3d70e20f12f11d2768e401b8114aa657f portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7836672-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F2220b5a771141f6ca7f14ed4462ae3b3d70e20f12f11d2768e401b8114aa657f\ntime=2026-09-16T18:02:16.065Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded"}},{"row_id":"eb2f58caadfa6b0b4c382d2d983df812fe0648bc14bb784cefa3a90f2156d63e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100270038647836672","occurred_at":"2026-09-16T17:06:12Z","failed_at":"alert","signature_id":"659a105790bd191ed63ba5bb7e543dad17b2528556030da7e70281aa38365890","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T17:55:32Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7836672-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7836672/providers/microsoft.monitor/accounts/services-j7836672\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5979b8454d-zdqrc\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-16T17:55:32Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7836672-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7836672/providers/microsoft.monitor/accounts/services-j7836672\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5979b8454d-zdqrc\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100268367708426240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6969,"pr_state":"open","pr_sha":"c4046b5e91e6786001b5754f90d8570f950ec5bd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T16:59:34Z","started_at":"2026-09-16T16:59:34Z","completed_at":"2026-09-16T18:54:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:01.349824801Z","timing_metadata_checked_at":"2026-09-18T15:41:01.349824801Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:01.352704811Z","region_metadata_checked_at":"2026-09-18T15:41:01.352704811Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"09c4ae1ef27ea0f7d6f10a1393776dcf36b6d1e13ac94a1787a5faae65ff5eb1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100268367708426240","occurred_at":"2026-09-16T16:59:34Z","failed_at":"alert","signature_id":"4aa09e305fda5dad99f7aa547324f1cc9a5313a1ed785f8832369357abebfa2a","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T18:20:28Z\n Ended: 2026-09-16T18:28:31Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j8426240-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-6r74z\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-6r74z (namespace arobit) on cluster ci01-j8426240-mgmt-2 has averaged 162.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T18:20:28Z\n Ended: 2026-09-16T18:32:31Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j8426240-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-6c87p\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-6c87p (namespace arobit) on cluster ci01-j8426240-svc has averaged 178.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T18:20:28Z\n Ended: 2026-09-16T18:32:31Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j8426240-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-56d8b8b5b4-b5s2q\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-56d8b8b5b4-b5s2q (namespace clusters-service) on cluster ci01-j8426240-svc has averaged 236.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"38352c761ed809ff31a8f4063fd3c8c13c65cef715f01035ba6a7f1e76012289","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100268367708426240","occurred_at":"2026-09-16T16:59:34Z","failed_at":"e2e","signature_id":"11b5fac45f89b3b06fa90df1c77a750f6663c0d038abc00b0bfa209370a40fa8","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bb5820\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-xxwwr4bjs848\" to finish creating: GET https://rp.j8426240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/13d2af80-0cd5-4fee-b2fe-b55a2b2248e3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/13d2af80-0cd5-4fee-b2fe-b55a2b2248e3\",\n \"name\": \"13d2af80-0cd5-4fee-b2fe-b55a2b2248e3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T18:00:53.783577799Z\",\n \"endTime\": \"2026-09-16T18:19:53.25990245Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bb5820\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-xxwwr4bjs848\" to finish creating: GET https://rp.j8426240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/13d2af80-0cd5-4fee-b2fe-b55a2b2248e3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/13d2af80-0cd5-4fee-b2fe-b55a2b2248e3\",\n \"name\": \"13d2af80-0cd5-4fee-b2fe-b55a2b2248e3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T18:00:53.783577799Z\",\n \"endTime\": \"2026-09-16T18:19:53.25990245Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6384be86b4732f9176967ccd72b7f15c54c80f1de41250fd9df4b3bce4fff232","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100268367708426240","occurred_at":"2026-09-16T16:59:34Z","failed_at":"e2e","signature_id":"131d9678ec571fa272cfc7362879effebfa9b5e73d92d8d4799c9e8c4dbac244","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--prf7fs699p2j/cluster-hs-pre-shc7r5 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008a9140\u003e: \n failed to create HCP cluster cluster-hs-pre-shc7r5: failed waiting for cluster=\"cluster-hs-pre-shc7r5\" in resourcegroup=\"rg-hs-pre--prf7fs699p2j\" to finish creating: GET https://rp.j8426240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74928ae1-e11f-4f60-a3b5-4c81b61651aa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74928ae1-e11f-4f60-a3b5-4c81b61651aa\",\n \"name\": \"74928ae1-e11f-4f60-a3b5-4c81b61651aa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T17:46:42.149025687Z\",\n \"endTime\": \"2026-09-16T18:15:43.737568191Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m6s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--prf7fs699p2j/cluster-hs-pre-shc7r5 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008a9140\u003e: \n failed to create HCP cluster cluster-hs-pre-shc7r5: failed waiting for cluster=\"cluster-hs-pre-shc7r5\" in resourcegroup=\"rg-hs-pre--prf7fs699p2j\" to finish creating: GET https://rp.j8426240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74928ae1-e11f-4f60-a3b5-4c81b61651aa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74928ae1-e11f-4f60-a3b5-4c81b61651aa\",\n \"name\": \"74928ae1-e11f-4f60-a3b5-4c81b61651aa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T17:46:42.149025687Z\",\n \"endTime\": \"2026-09-16T18:15:43.737568191Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m6s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"68bc0539e447ae90db67dd62442833957fc0ef220ee1995ba10e0a735186a319","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100268367708426240","occurred_at":"2026-09-16T16:59:34Z","failed_at":"e2e","signature_id":"ee111baab05f6b8ee26f353e865437e1bffda7140d9fc154398f141cd776e6f7","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n \u003cbool\u003e: ...\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}},{"row_id":"fb28b085d2a0da1d40061dd4ddc81e400d2d8e8ca4925185b391307b1d9fb08a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100268367708426240","occurred_at":"2026-09-16T16:59:34Z","failed_at":"alert","signature_id":"e4b9c749f68fadead7aeb206b02a1f85e9ae2c2c7142b04d727f3089de1f8541","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T17:58:32Z\n Ended: 2026-09-16T18:19:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8426240-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss8g0v48bf4djgpg9nc4h091itrlbc2-m3n4m9t6z0r3g6f\", pod=\"router-689655689c-bsfzz\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss8g0v48bf4djgpg9nc4h091itrlbc2-m3n4m9t6z0r3g6f/router-689655689c-bsfzz has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100265336732389376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6965,"pr_state":"closed","pr_sha":"ac0717465df81eeeb7d8446cc4182a603bb8004f","final_merged_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T16:47:31Z","started_at":"2026-09-16T16:47:31Z","completed_at":"2026-09-16T17:19:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:59.652471193Z","timing_metadata_checked_at":"2026-09-18T15:40:59.652471193Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:59.655681705Z","region_metadata_checked_at":"2026-09-18T15:40:59.655681705Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"894e4363e56dcd70d796b08f3ce5c96f8265526569c2347b36bb85b4642acbde","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100265336732389376","occurred_at":"2026-09-16T16:47:31Z","failed_at":"provision","signature_id":"2a5637582a78b2337c484f5a7a9ff7f9d84aa702c7b086e89182f1549eb13d57","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Monitoring/regional/monitoring","test_suite":"step graph","failure_text":"time=2026-09-16T17:10:05.170Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring description=\"Step monitoring\\n Kind: ARMStack\\n Template: templates/monitoring.bicep\\n Parameters: configurations/monitoring.tmpl.bicepparam\"\ntime=2026-09-16T17:11:24.952Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring err=\"failed to run ARM step: failed to wait for deployment stack at resource group scope: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/canadacentral/deploymentStackOperationStatus/aeca7e8d-50c1-47d0-a644-b291a00162ea\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentStackDeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/canadacentral/deploymentStackOperationStatus/aeca7e8d-50c1-47d0-a644-b291a00162ea\\\",\\n \\\"name\\\": \\\"aeca7e8d-50c1-47d0-a644-b291a00162ea\\\",\\n \\\"status\\\": \\\"failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentStackDeploymentFailed\\\",\\n \\\"message\\\": \\\"One or more resources could not be deployed. Correlation id: 'df67a523-14d6-42a2-9636-444dbb7d2763'.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2389376/providers/Microsoft.Resources/deployments/nbyngfosfawbsx5vtxb2uo6lmfim2rbaopp3emm2zwlmm-260916172yp3z\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"Property name: Rules[0].Expression, Attempted value: null, Error: 1:237: parse error: unknown escape sequence U+002E '.' Activity ID: 91890f9b-76b2-4791-9c30-927a7253a007.\\\"\\n }\\n ]\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e405dc3ba5eaa76b269136ece3c7ec09bc9d5c543da581d8b5e14a4da0eb2410","failure_pattern":"ERROR CODE: DeploymentStackDeploymentFailed","search_query":"ERROR CODE: DeploymentStackDeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e405dc3ba5eaa76b269136ece3c7ec09bc9d5c543da581d8b5e14a4da0eb2410"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100263412582846464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6966,"pr_state":"closed","pr_sha":"c93ff943c35ad2d21e3a98df5a27db691c9749a7","final_merged_sha":"c93ff943c35ad2d21e3a98df5a27db691c9749a7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T16:39:53Z","started_at":"2026-09-16T16:39:53Z","completed_at":"2026-09-16T18:59:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:00.169354352Z","timing_metadata_checked_at":"2026-09-18T15:41:00.169354352Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:00.172546364Z","region_metadata_checked_at":"2026-09-18T15:41:00.172546364Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"51bb79b4a8e89bc323c8d33075d7eb6efce85b576e175e84aa77b448818c87a1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100263412582846464","occurred_at":"2026-09-16T16:39:53Z","failed_at":"alert","signature_id":"f812f1c6853ffe5ec0bbae33597f7ac343264a944ebc861a726251e8dce290ce","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T18:21:59Z\n Ended: 2026-09-16T18:42:00Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2846464-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.181:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2846464/providers/microsoft.monitor/accounts/hcps-j2846464\", namespace=\"ocm-arohcpci01-2ss8i74sks91slsm7oliign0ld9qcsch-q0f6x6b6a5y9c2t\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-fbvzq\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2ss8i74sks91slsm7oliign0ld9qcsch-q0f6x6b6a5y9c2t/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-16T18:21:59Z\n Ended: 2026-09-16T18:29:59Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2846464-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.181:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2846464/providers/microsoft.monitor/accounts/hcps-j2846464\", namespace=\"ocm-arohcpci01-2ss8i74sks91slsm7oliign0ld9qcsch-q0f6x6b6a5y9c2t\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-fbvzq\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2ss8i74sks91slsm7oliign0ld9qcsch-q0f6x6b6a5y9c2t/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"5d2491fc07c5e3a8d0dcfc0aedb1e1ff181236bc66c2530d89491b46bf911346","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100263412582846464","occurred_at":"2026-09-16T16:39:53Z","failed_at":"e2e","signature_id":"4c86e6598dc5b904362415f87179a0f883f34b4b44e009896f6a7f0de46090e2","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-vlzgbk with version 4.21.33\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006bee60\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-vlzgbk: failed waiting for cluster=\"np-version-upgrade-cluster-vlzgbk\" in resourcegroup=\"rg-np-version-upgrade-vlzgbk-6gtpz2tlsds2\" to finish creating: GET https://rp.j2846464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83de4d68-f488-460e-8446-69209da0d691\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83de4d68-f488-460e-8446-69209da0d691\",\n \"name\": \"83de4d68-f488-460e-8446-69209da0d691\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T17:51:23.682356624Z\",\n \"endTime\": \"2026-09-16T18:20:27.523942323Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-vlzgbk with version 4.21.33\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006bee60\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-vlzgbk: failed waiting for cluster=\"np-version-upgrade-cluster-vlzgbk\" in resourcegroup=\"rg-np-version-upgrade-vlzgbk-6gtpz2tlsds2\" to finish creating: GET https://rp.j2846464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83de4d68-f488-460e-8446-69209da0d691\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83de4d68-f488-460e-8446-69209da0d691\",\n \"name\": \"83de4d68-f488-460e-8446-69209da0d691\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T17:51:23.682356624Z\",\n \"endTime\": \"2026-09-16T18:20:27.523942323Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5d789210f6f9ce4fad504089502ce86836c22954200892296bd173c38061a94e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100263412582846464","occurred_at":"2026-09-16T16:39:53Z","failed_at":"alert","signature_id":"18090d26c3a2666e0dfd30f6dc97088df512304fef09dec58c535d88cec6f5bd","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T18:12:57Z\n Ended: 2026-09-16T18:25:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2846464-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.181:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2846464/providers/microsoft.monitor/accounts/hcps-j2846464\", namespace=\"ocm-arohcpci01-2ss8i74sks91slsm7oliign0ld9qcsch-q0f6x6b6a5y9c2t\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"6e854687-f7c2-439d-b129-bec4fba1965e\"\n Description: Pod ocm-arohcpci01-2ss8i74sks91slsm7oliign0ld9qcsch-q0f6x6b6a5y9c2t/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-09-16T18:12:57Z\n Ended: 2026-09-16T18:32:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2846464-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.181:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2846464/providers/microsoft.monitor/accounts/hcps-j2846464\", namespace=\"ocm-arohcpci01-2ss8i74sks91slsm7oliign0ld9qcsch-q0f6x6b6a5y9c2t\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"6e854687-f7c2-439d-b129-bec4fba1965e\"\n Description: Pod ocm-arohcpci01-2ss8i74sks91slsm7oliign0ld9qcsch-q0f6x6b6a5y9c2t/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"7325c8fa313562e5d2fd251d9126dccad048c6cf6aeda1e5f2c238bf2d52b197","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100263412582846464","occurred_at":"2026-09-16T16:39:53Z","failed_at":"alert","signature_id":"e80538656acc5e08b78c0da3cbda2f5f642ff686a707fd278f435ee95c791d0f","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T18:24:09Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2846464-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-f75b695b-zmfgv\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-f75b695b-zmfgv (namespace mgmt-agent) on cluster ci01-j2846464-mgmt-2 has averaged 282.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T18:24:09Z\n Ended: 2026-09-16T18:48:14Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2846464-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-m5d7l\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-m5d7l (namespace arobit) on cluster ci01-j2846464-mgmt-2 has averaged 287% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"7e343013d2a1b95cc30bb7f5ede278cc1d60ab32c19a1616361f6da5695920ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100263412582846464","occurred_at":"2026-09-16T16:39:53Z","failed_at":"e2e","signature_id":"dcd0bfb37a59db9bd0b774c26b5a47b2e4002b0f9d6d7d9a6292d0c9dca75bea","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:407]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc002e66048\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:407]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc002e66048\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100262131709513728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6928,"pr_state":"open","pr_sha":"dde72f49803d49335aafd35b01b79b6a92ac9d27","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T16:34:47Z","started_at":"2026-09-16T16:34:47Z","completed_at":"2026-09-16T18:29:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:41.705744036Z","timing_metadata_checked_at":"2026-09-18T15:40:41.705744036Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:41.709304549Z","region_metadata_checked_at":"2026-09-18T15:40:41.709304549Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"872cc73786d97ad3947436af2f59db9010663def036b34a7ed6768a1ca78c8b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100262131709513728","occurred_at":"2026-09-16T16:34:47Z","failed_at":"alert","signature_id":"8bca23390b6e128f806583b6f43999dfe3e0891c3867f9b00f777f7b70551bad","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T17:52:05Z\n Ended: 2026-09-16T18:06:07Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9513728-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-6f85fb4496-67kgn\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-6f85fb4496-67kgn (namespace mgmt-agent) on cluster ci01-j9513728-mgmt-1 has averaged 176.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T17:52:05Z\n Ended: 2026-09-16T18:04:06Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9513728-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-kpqlb\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-kpqlb (namespace arobit) on cluster ci01-j9513728-mgmt-1 has averaged 156% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"9d7f91b59c822d3c2dc8bdb3ad87fac14e110f5a127b0f4a1c8c571ff289aafe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100262131709513728","occurred_at":"2026-09-16T16:34:47Z","failed_at":"alert","signature_id":"b7c6ba500fe6358c309a0b082682885bdc0c4016fb309f71ff96ef107142bcca","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T17:30:40Z\n Ended: 2026-09-16T18:00:38Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9513728-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss82miun3vj5m88let2469mn6t6p3gi-v20261001\", pod=\"router-5c4f9947bb-vqdfr\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss82miun3vj5m88let2469mn6t6p3gi-v20261001/router-5c4f9947bb-vqdfr has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"a71d57d9163fda0cee4e1b3c19f5388eb7ab39eb1076e72cea03395535fdff91","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100262131709513728","occurred_at":"2026-09-16T16:34:47Z","failed_at":"e2e","signature_id":"7516a34fe822a219e86822cf9a1b8d51e620d585523f3da6508b9fb567e9b286","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:98]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c626c0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-wrwln5phv4nq\" to finish creating: GET https://rp.j9513728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/000400be-8a8d-4f23-afdd-984e12bfed12\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/000400be-8a8d-4f23-afdd-984e12bfed12\",\n \"name\": \"000400be-8a8d-4f23-afdd-984e12bfed12\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T17:18:18.426850262Z\",\n \"endTime\": \"2026-09-16T17:47:24.672789171Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m0s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:98]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c626c0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-wrwln5phv4nq\" to finish creating: GET https://rp.j9513728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/000400be-8a8d-4f23-afdd-984e12bfed12\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/000400be-8a8d-4f23-afdd-984e12bfed12\",\n \"name\": \"000400be-8a8d-4f23-afdd-984e12bfed12\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T17:18:18.426850262Z\",\n \"endTime\": \"2026-09-16T17:47:24.672789171Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m0s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d0a0aeba0add719a0d0f802c737c5485d44edb7a38443f0cd71c4726fb910def","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100262131709513728","occurred_at":"2026-09-16T16:34:47Z","failed_at":"e2e","signature_id":"4d2c343c16f83436cc3986db89b356fdfb3e28de9ebaa9ceda587e5572a6ffc6","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-4k9m7kr6rf4h/cluster-candidate-4-21-76dltv should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00116e240\u003e: \n failed to create HCP cluster cluster-candidate-4-21-76dltv: failed waiting for cluster=\"cluster-candidate-4-21-76dltv\" in resourcegroup=\"rg-candidate-4-21-4k9m7kr6rf4h\" to finish creating: GET https://rp.j9513728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d958346-0f65-40cd-b391-5426f09786f1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d958346-0f65-40cd-b391-5426f09786f1\",\n \"name\": \"7d958346-0f65-40cd-b391-5426f09786f1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T17:18:18.739919439Z\",\n \"endTime\": \"2026-09-16T17:47:24.772096699Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m43s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-4k9m7kr6rf4h/cluster-candidate-4-21-76dltv should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00116e240\u003e: \n failed to create HCP cluster cluster-candidate-4-21-76dltv: failed waiting for cluster=\"cluster-candidate-4-21-76dltv\" in resourcegroup=\"rg-candidate-4-21-4k9m7kr6rf4h\" to finish creating: GET https://rp.j9513728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d958346-0f65-40cd-b391-5426f09786f1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d958346-0f65-40cd-b391-5426f09786f1\",\n \"name\": \"7d958346-0f65-40cd-b391-5426f09786f1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T17:18:18.739919439Z\",\n \"endTime\": \"2026-09-16T17:47:24.772096699Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m43s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100260298874163200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6977,"pr_state":"closed","pr_sha":"557c04379e033b9de0bd85904b49103f2cdef1b1","final_merged_sha":"557c04379e033b9de0bd85904b49103f2cdef1b1","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T16:27:30Z","started_at":"2026-09-16T16:27:30Z","completed_at":"2026-09-16T18:24:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:06.018406212Z","timing_metadata_checked_at":"2026-09-18T15:41:06.018406212Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:06.021322223Z","region_metadata_checked_at":"2026-09-18T15:41:06.021322223Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"797b71169791b88228195bd60564de2e0b6e22086052c671174b2b3933f03fe2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100260298874163200","occurred_at":"2026-09-16T16:27:30Z","failed_at":"alert","signature_id":"641a1726bac71eedb2360eec9a405c0d1e826983dcd4ea335ba7c7dfa3cebe9d","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 7 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T17:35:12Z\n Ended: 2026-09-16T17:50:10Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4163200-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-j2smn\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-j2smn (namespace arobit) on cluster ci01-j4163200-svc has averaged 173.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T17:36:10Z\n Ended: 2026-09-16T17:50:10Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4163200-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-6fbd676998-lwslc\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-6fbd676998-lwslc (namespace aro-hcp) on cluster ci01-j4163200-svc has averaged 365.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T17:37:10Z\n Ended: 2026-09-16T17:47:11Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4163200-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-6bb57589d5-9gf7j\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-6bb57589d5-9gf7j (namespace clusters-service) on cluster ci01-j4163200-svc has averaged 173.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-16T17:37:10Z\n Ended: 2026-09-16T17:50:10Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4163200-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-6bb57589d5-r4swh\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-6bb57589d5-r4swh (namespace clusters-service) on cluster ci01-j4163200-svc has averaged 645.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-16T17:37:10Z\n Ended: 2026-09-16T17:43:12Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4163200-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-6bb57589d5-gh9w9\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-6bb57589d5-gh9w9 (namespace clusters-service) on cluster ci01-j4163200-svc has averaged 178% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Resolved\n Started: 2026-09-16T17:44:11Z\n Ended: 2026-09-16T17:53:10Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4163200-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-fdff466bc-5kkx2\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-fdff466bc-5kkx2 (namespace mgmt-agent) on cluster ci01-j4163200-mgmt-1 has averaged 150.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 7:\n State: Resolved\n Started: 2026-09-16T17:48:08Z\n Ended: 2026-09-16T17:58:08Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4163200-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-kqxdq\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-kqxdq (namespace arobit) on cluster ci01-j4163200-mgmt-1 has averaged 140.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100260058020450304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6856,"pr_state":"closed","pr_sha":"7ca88f2e65efb54f23911ebd52997959c6e9b500","final_merged_sha":"96f38fd5a277adea0aeec323ef5685124af19024","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T16:26:33Z","started_at":"2026-09-16T16:26:33Z","completed_at":"2026-09-16T18:29:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:28.501113833Z","timing_metadata_checked_at":"2026-09-18T15:40:28.501113833Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:28.504874946Z","region_metadata_checked_at":"2026-09-18T15:40:28.504874946Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"d291ec1f3b03975f0cdf8a4d95df239c4d613b227012a91f37cbe51e7b8d8352","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100260058020450304","occurred_at":"2026-09-16T16:26:33Z","failed_at":"alert","signature_id":"84adc99394ffdde58d8777dabe660ecf4888bc0533bab685e83dea23a8c7fc1a","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T17:39:57Z\n Ended: 2026-09-16T17:45:56Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j0450304-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j0450304-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"fbacfa719527cc0c044e713cea99b76e4231dcb40c4733d6023a34bf521d3cd2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100260058020450304","occurred_at":"2026-09-16T16:26:33Z","failed_at":"alert","signature_id":"9a105f0d73b76a92d17a2de5a3542ed76121642b109339640b150bdbd8718601","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 6 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T17:44:57Z\n Ended: 2026-09-16T17:54:55Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0450304-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-8rhn6\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-8rhn6 (namespace arobit) on cluster ci01-j0450304-mgmt-1 has averaged 243.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T17:44:57Z\n Ended: 2026-09-16T17:56:54Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0450304-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-84ccc6569-wlml9\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-84ccc6569-wlml9 (namespace aro-hcp) on cluster ci01-j0450304-svc has averaged 450.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T17:44:57Z\n Ended: 2026-09-16T17:56:54Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0450304-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-65d5c9b4f7-r55zf\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-65d5c9b4f7-r55zf (namespace clusters-service) on cluster ci01-j0450304-svc has averaged 891% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-16T17:44:57Z\n Ended: 2026-09-16T17:51:56Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0450304-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-trvfl\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-trvfl (namespace arobit) on cluster ci01-j0450304-mgmt-2 has averaged 148.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-16T17:44:57Z\n Ended: 2026-09-16T17:56:54Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0450304-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-56f9n\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-56f9n (namespace arobit) on cluster ci01-j0450304-svc has averaged 264.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Resolved\n Started: 2026-09-16T17:44:57Z\n Ended: 2026-09-16T17:51:56Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0450304-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-787f7cfb65-zfhwc\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-787f7cfb65-zfhwc (namespace mgmt-agent) on cluster ci01-j0450304-mgmt-2 has averaged 166.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100253677548736512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6976,"pr_state":"open","pr_sha":"3f37770fba22a55b4336e3a85cbf275bf37c8279","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T16:01:02Z","started_at":"2026-09-16T16:01:02Z","completed_at":"2026-09-16T18:17:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:04.88190312Z","timing_metadata_checked_at":"2026-09-18T15:41:04.88190312Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:04.885487233Z","region_metadata_checked_at":"2026-09-18T15:41:04.885487233Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"21ea6fe71c5a0b652635c7fff9b375949c2fb9372ea57111687f01d319c2c4ab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100253677548736512","occurred_at":"2026-09-16T16:01:02Z","failed_at":"alert","signature_id":"e26ae0379e9c2f78baac16193a817ee1d4bd6be52f5b22f001aad146994821f2","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T17:28:09Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j8736512-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-5d75778c4b-xzbtr\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-5d75778c4b-xzbtr (namespace clusters-service) on cluster ci01-j8736512-svc has averaged 655.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T17:28:09Z\n Ended: 2026-09-16T17:40:12Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j8736512-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-5d75778c4b-hc6kg\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-5d75778c4b-hc6kg (namespace clusters-service) on cluster ci01-j8736512-svc has averaged 197.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T17:28:09Z\n Ended: 2026-09-16T17:35:11Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j8736512-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-6777c455fd-5fq84\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-6777c455fd-5fq84 (namespace mgmt-agent) on cluster ci01-j8736512-mgmt-1 has averaged 152.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-16T17:37:10Z\n Ended: 2026-09-16T17:44:12Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j8736512-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-cvrzx\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-cvrzx (namespace arobit) on cluster ci01-j8736512-mgmt-2 has averaged 127.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100253157446651904","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6980,"pr_state":"closed","pr_sha":"4c6ef86c4410d1e48dd7653c177a4370cf38a246","final_merged_sha":"4c6ef86c4410d1e48dd7653c177a4370cf38a246","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T15:59:08Z","started_at":"2026-09-16T15:59:08Z","completed_at":"2026-09-16T18:17:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:07.444465647Z","timing_metadata_checked_at":"2026-09-18T15:41:07.444465647Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:07.44808916Z","region_metadata_checked_at":"2026-09-18T15:41:07.44808916Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"21cc5500199ca785b0ecdb9c93b058a8a8d336f785caaaa653dbd84ae6806302","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100253157446651904","occurred_at":"2026-09-16T15:59:08Z","failed_at":"alert","signature_id":"bb91f870f2d6651db04eb26377a6ab0f560e56353470294354ffb9ea8c0407f9","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 8 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T17:27:30Z\n Ended: 2026-09-16T17:45:32Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6651904-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-5f5c4b79d-nzh8s\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-5f5c4b79d-nzh8s (namespace mgmt-agent) on cluster ci01-j6651904-mgmt-1 has averaged 195.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T17:27:30Z\n Ended: 2026-09-16T17:45:32Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6651904-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-qwq7m\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-qwq7m (namespace arobit) on cluster ci01-j6651904-mgmt-1 has averaged 191.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-16T17:27:30Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6651904-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-565bdd9568-x62g9\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-565bdd9568-x62g9 (namespace clusters-service) on cluster ci01-j6651904-svc has averaged 720.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-16T17:27:30Z\n Ended: 2026-09-16T17:45:32Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6651904-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-59bbbb688b-f8hwv\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-59bbbb688b-f8hwv (namespace aro-hcp) on cluster ci01-j6651904-svc has averaged 370.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-16T17:27:30Z\n Ended: 2026-09-16T17:44:32Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6651904-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-bhgm5\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-bhgm5 (namespace arobit) on cluster ci01-j6651904-mgmt-2 has averaged 175.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Resolved\n Started: 2026-09-16T17:27:30Z\n Ended: 2026-09-16T17:41:35Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6651904-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-vkvcx\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-vkvcx (namespace arobit) on cluster ci01-j6651904-svc has averaged 154.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 7:\n State: Resolved\n Started: 2026-09-16T17:27:30Z\n Ended: 2026-09-16T17:45:32Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6651904-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-565bdd9568-mf8jt\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-565bdd9568-mf8jt (namespace clusters-service) on cluster ci01-j6651904-svc has averaged 204% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 8:\n State: Resolved\n Started: 2026-09-16T17:31:31Z\n Ended: 2026-09-16T17:47:31Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6651904-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-565bdd9568-m4fxr\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-565bdd9568-m4fxr (namespace clusters-service) on cluster ci01-j6651904-svc has averaged 262.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252605216198656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6933,"pr_state":"closed","pr_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","final_merged_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T15:56:55Z","started_at":"2026-09-16T15:56:55Z","completed_at":"2026-09-16T17:55:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:43.528654397Z","timing_metadata_checked_at":"2026-09-18T15:40:43.528654397Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:43.531931609Z","region_metadata_checked_at":"2026-09-18T15:40:43.531931609Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"04489e42003a43fcafdeafe09ed44ff14c35f5a58a11120703d56a0cc43f0223","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252605216198656","occurred_at":"2026-09-16T15:56:55Z","failed_at":"e2e","signature_id":"d9283aaa1d82489c4a0f2f12dd912bceba77f21d6bae60f6a616ab523d7ace51","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-z9mjpt\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010173e0\u003e: \n DNS for route host agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-z9mjpt\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010173e0\u003e: \n DNS for route host agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded"}},{"row_id":"0e7d4600811777b30dd3bfee61c773a1898664813ebd8ba16012249cc3d450f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252605216198656","occurred_at":"2026-09-16T15:56:55Z","failed_at":"e2e","signature_id":"662575cd71f8c3606ecf799e2d76b50f1b3ecb8c083205f994e08e750a8a40a2","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:115]: failed to verify simple web app runs on cluster \"cluster-hs-pre-r67qjj\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000814820\u003e: \n DNS for route host agnhost-e2e-sample-app-zxgt9.apps.aro.j0p9f5n7c5x3h1g.7jfd.j6198656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-zxgt9.apps.aro.j0p9f5n7c5x3h1g.7jfd.j6198656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-zxgt9.apps.aro.j0p9f5n7c5x3h1g.7jfd.j6198656.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:115]: failed to verify simple web app runs on cluster \"cluster-hs-pre-r67qjj\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000814820\u003e: \n DNS for route host agnhost-e2e-sample-app-zxgt9.apps.aro.j0p9f5n7c5x3h1g.7jfd.j6198656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-zxgt9.apps.aro.j0p9f5n7c5x3h1g.7jfd.j6198656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-zxgt9.apps.aro.j0p9f5n7c5x3h1g.7jfd.j6198656.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded"}},{"row_id":"6e5570c5edee7408ade12ee23658b96aec14c196d361a8df997780305b089e42","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252605216198656","occurred_at":"2026-09-16T15:56:55Z","failed_at":"e2e","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:170]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred. Got:\n \u003cnil\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:170]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred. Got:\n \u003cnil\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"406d47e638f407d24104655febf18e753de33546f725682f49558d9bc4ae2365","failure_pattern":"private ingress should not be reachable from outside the VNet, but connection succeeded","search_query":"private ingress should not be reachable from outside the VNet, but connection succeeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["406d47e638f407d24104655febf18e753de33546f725682f49558d9bc4ae2365","c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252538870697984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6791,"pr_state":"closed","pr_sha":"764ee60fa32d166f92c6c286f96007787ee189de","final_merged_sha":"00c5d478babb26a92f08e45ee9f4c322bafd7507","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T15:56:40Z","started_at":"2026-09-16T15:56:40Z","completed_at":"2026-09-16T18:01:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:23.095468784Z","timing_metadata_checked_at":"2026-09-18T15:40:23.095468784Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:23.099275298Z","region_metadata_checked_at":"2026-09-18T15:40:23.099275298Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"45e83e2357999e84524191290ccb18f290c756e2267dfbc4c330a082c91845cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252538870697984","occurred_at":"2026-09-16T15:56:40Z","failed_at":"alert","signature_id":"984ec4e1000694cd92d63a32c0f7a6e5d7b7f677e37cfce55ad670237d8b5da0","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T17:31:16Z\n Ended: 2026-09-16T17:42:17Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0697984-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-7b4b7c85b9-5w7qb\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-7b4b7c85b9-5w7qb (namespace clusters-service) on cluster ci01-j0697984-svc has averaged 629.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"c87c2b53f05d509e6c55ac275950a02f04cadb82640a5d55acb40632285310e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252538870697984","occurred_at":"2026-09-16T15:56:40Z","failed_at":"e2e","signature_id":"b7562eb1317f094fe7e5e1b3fd0e3859a05304435b1ff984fb6d716f402b4c7c","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.005s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001ffe8a0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \"0IciLMVmLbqimBoU8f5q6QBEpmWjuJNgDzWaCM2v9MM=\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.005s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001ffe8a0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \"0IciLMVmLbqimBoU8f5q6QBEpmWjuJNgDzWaCM2v9MM=\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fac5ab822da5a4db89037217494583b4069bba01be61d8f3a5122bc721b483e9","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \"0IciLMVmLbqimBoU8f5q6QBEpmWjuJNgDzWaCM2v9MM=\"","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \"0IciLMVmLbqimBoU8f5q6QBEpmWjuJNgDzWaCM2v9MM=\""}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","fac5ab822da5a4db89037217494583b4069bba01be61d8f3a5122bc721b483e9"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252533153861632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6964,"pr_state":"closed","pr_sha":"af707ce72df7d034eaee96beafbefdfe37fc0800","final_merged_sha":"af707ce72df7d034eaee96beafbefdfe37fc0800","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T15:56:39Z","started_at":"2026-09-16T15:56:39Z","completed_at":"2026-09-16T18:16:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:58.766424406Z","timing_metadata_checked_at":"2026-09-18T15:40:58.766424406Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:58.770453021Z","region_metadata_checked_at":"2026-09-18T15:40:58.770453021Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"1f1b139511dca64237440041bf42550027b9f2495eb1d73573c696c0429758e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252533153861632","occurred_at":"2026-09-16T15:56:39Z","failed_at":"alert","signature_id":"59e4de1f8e513f5cc08bd4e593ae29483ec56ee54e8e92faff2ba25b8713cc09","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T17:26:31Z\n Ended: 2026-09-16T17:39:34Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3861632-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-5f5df44c46-k8k9r\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-5f5df44c46-k8k9r (namespace mgmt-agent) on cluster ci01-j3861632-mgmt-1 has averaged 139.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T17:26:31Z\n Ended: 2026-09-16T17:32:32Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3861632-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-5f5df44c46-tlbzj\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-5f5df44c46-tlbzj (namespace mgmt-agent) on cluster ci01-j3861632-mgmt-2 has averaged 144.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T17:29:31Z\n Ended: 2026-09-16T17:47:31Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3861632-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-b9nwv\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-b9nwv (namespace arobit) on cluster ci01-j3861632-mgmt-1 has averaged 142.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-16T17:33:32Z\n Ended: 2026-09-16T17:41:30Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3861632-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-4lnj7\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-4lnj7 (namespace arobit) on cluster ci01-j3861632-mgmt-2 has averaged 141.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"440e96887354f7418aa8f000366287547b72b39d4d6e17052e59eac1766618fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252533153861632","occurred_at":"2026-09-16T15:56:39Z","failed_at":"alert","signature_id":"eb143e51cc4e677a9811a011abde7f9846dfb86171fd5089044d86a86bc3c30e","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T17:27:30Z\n Ended: 2026-09-16T17:34:29Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3861632-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3861632-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"6516cb920637e41c24139c86dc700076c68c7d4d6944085df183df695458625b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252533153861632","occurred_at":"2026-09-16T15:56:39Z","failed_at":"e2e","signature_id":"8fab9814450d28b4f592dfdb0cd8c2267f5fe8d7043a907d74c9a720713cb056","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-5xvglfvdmmsk/cluster-candidate-4-20-8wb96p should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b6a380\u003e: \n failed to create HCP cluster cluster-candidate-4-20-8wb96p: failed waiting for cluster=\"cluster-candidate-4-20-8wb96p\" in resourcegroup=\"rg-candidate-4-20-5xvglfvdmmsk\" to finish creating: GET https://rp.j3861632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2c19faf7-c0c3-42d6-a653-b3ad4df2d164\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2c19faf7-c0c3-42d6-a653-b3ad4df2d164\",\n \"name\": \"2c19faf7-c0c3-42d6-a653-b3ad4df2d164\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T16:55:05.290922061Z\",\n \"endTime\": \"2026-09-16T17:24:07.163114799Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.38 is Partial (want Completed), started 16m23s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-5xvglfvdmmsk/cluster-candidate-4-20-8wb96p should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b6a380\u003e: \n failed to create HCP cluster cluster-candidate-4-20-8wb96p: failed waiting for cluster=\"cluster-candidate-4-20-8wb96p\" in resourcegroup=\"rg-candidate-4-20-5xvglfvdmmsk\" to finish creating: GET https://rp.j3861632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2c19faf7-c0c3-42d6-a653-b3ad4df2d164\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2c19faf7-c0c3-42d6-a653-b3ad4df2d164\",\n \"name\": \"2c19faf7-c0c3-42d6-a653-b3ad4df2d164\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T16:55:05.290922061Z\",\n \"endTime\": \"2026-09-16T17:24:07.163114799Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.38 is Partial (want Completed), started 16m23s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8611820c8181692bb90e562c6d1310a7d6f59f596acb46c35337148cd831cc6c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252533153861632","occurred_at":"2026-09-16T15:56:39Z","failed_at":"e2e","signature_id":"d2849a0a17cc4329034ec17d0d9ec1983dfbaf5324ca580cd9753b318ae0a9bc","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bb8ce0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-r56zm8qhbr8l\" to finish creating: GET https://rp.j3861632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/590c2a93-60a1-4c63-b56e-6b6322a7d814\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/590c2a93-60a1-4c63-b56e-6b6322a7d814\",\n \"name\": \"590c2a93-60a1-4c63-b56e-6b6322a7d814\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T16:55:05.068738086Z\",\n \"endTime\": \"2026-09-16T17:24:07.383701828Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 16m27s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bb8ce0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-r56zm8qhbr8l\" to finish creating: GET https://rp.j3861632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/590c2a93-60a1-4c63-b56e-6b6322a7d814\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/590c2a93-60a1-4c63-b56e-6b6322a7d814\",\n \"name\": \"590c2a93-60a1-4c63-b56e-6b6322a7d814\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T16:55:05.068738086Z\",\n \"endTime\": \"2026-09-16T17:24:07.383701828Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 16m27s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6889,"pr_state":"open","pr_sha":"c7331b711087e58d7a4c69ff0f50f20ed440e104","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T15:55:51Z","started_at":"2026-09-16T15:55:51Z","completed_at":"2026-09-16T18:15:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:32.791560463Z","timing_metadata_checked_at":"2026-09-18T15:40:32.791560463Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:32.794894575Z","region_metadata_checked_at":"2026-09-18T15:40:32.794894575Z"},"failed_at":["alert","e2e"],"failed_test_count":10,"occurrences":[{"row_id":"7a2506927d63323c6b5911520255f7c2db7405b59e8614b95add9355f9f28024","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","failed_at":"alert","signature_id":"660cafcaf8e83347ebad23ce568c8f5b0468c2f68c8566340ecac05676075f1d","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 5 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T17:00:59Z\n Ended: 2026-09-16T17:19:02Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4807552-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-09-16T17:00:59Z\n Ended: 2026-09-16T17:19:02Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4807552-svc\", component=\"backend\", name=\"datadump\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue datadump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 3:\n State: Resolved\n Started: 2026-09-16T17:00:59Z\n Ended: 2026-09-16T17:20:01Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4807552-svc\", component=\"backend\", name=\"csstatedump\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 4:\n State: Resolved\n Started: 2026-09-16T17:02:01Z\n Ended: 2026-09-16T17:19:02Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4807552-svc\", component=\"backend\", name=\"clusterclusterserviceupdatedispatch\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterclusterserviceupdatedispatch has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 5:\n State: Resolved\n Started: 2026-09-16T17:09:02Z\n Ended: 2026-09-16T17:19:02Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4807552-svc\", component=\"backend\", name=\"triggercontrolplaneupgrade\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue triggercontrolplaneupgrade has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"8bb0bed5fbd62a8fcf47338acfa3cae6676206319721bcc540a071ecd8f5a2ce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","failed_at":"e2e","signature_id":"5cf9c85c408d35be4deef4e7c087c454f1070d1509c8652940e0ebeb5471037c","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012b4030\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-sk5lk2xqrw7v\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/234e26dc-5c76-4335-9749-4a1c49aae81e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/234e26dc-5c76-4335-9749-4a1c49aae81e\",\n \"name\": \"234e26dc-5c76-4335-9749-4a1c49aae81e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T17:00:40.136241522Z\",\n \"endTime\": \"2026-09-16T17:24:42.83613595Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7is141l1pe3gtmk90b2cej7ltp3gn still exists (deletion dispatched at 2026-09-16T17:00:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012b4030\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-sk5lk2xqrw7v\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/234e26dc-5c76-4335-9749-4a1c49aae81e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/234e26dc-5c76-4335-9749-4a1c49aae81e\",\n \"name\": \"234e26dc-5c76-4335-9749-4a1c49aae81e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T17:00:40.136241522Z\",\n \"endTime\": \"2026-09-16T17:24:42.83613595Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7is141l1pe3gtmk90b2cej7ltp3gn still exists (deletion dispatched at 2026-09-16T17:00:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b2d9bdf10bd756a41af895df7f16c38d9b8ca7729eae2184e9c4375fba7db5fb","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9632323c0f28317b53ea82fa2792eed2f2a875ca73cf5e0afe4601d352ed3bf8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","failed_at":"e2e","signature_id":"c81fbcddae76e3d46ce74bd7eeec6dbb65a70aa3f34997838af6880b8bc4b471","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001488048\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-2jvpp5kvggw9\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8454bf1b-7bda-408c-958d-23419d4eddce\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8454bf1b-7bda-408c-958d-23419d4eddce\",\n \"name\": \"8454bf1b-7bda-408c-958d-23419d4eddce\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T16:56:41.886166734Z\",\n \"endTime\": \"2026-09-16T17:20:42.876661186Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7j11fl1lshknqe0flq48ksbqlst7s still exists (deletion dispatched at 2026-09-16T16:56:42Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001488048\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-2jvpp5kvggw9\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8454bf1b-7bda-408c-958d-23419d4eddce\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8454bf1b-7bda-408c-958d-23419d4eddce\",\n \"name\": \"8454bf1b-7bda-408c-958d-23419d4eddce\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T16:56:41.886166734Z\",\n \"endTime\": \"2026-09-16T17:20:42.876661186Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7j11fl1lshknqe0flq48ksbqlst7s still exists (deletion dispatched at 2026-09-16T16:56:42Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"997f70e35b8895590e27c7ce87332fca45edb643807ade56106f67de3813553f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","failed_at":"e2e","signature_id":"a552e8119646d9c743aa9a252024bde026b39e6cefa5828c2a635099d2355f2a","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0010ae0d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-gwq76w\" in resourcegroup=\"rg-zstream-upgrade-4-21-kkt25v682kzc\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a1f1fe55-fa55-489a-97de-ae14b6fbabf2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a1f1fe55-fa55-489a-97de-ae14b6fbabf2\",\n \"name\": \"a1f1fe55-fa55-489a-97de-ae14b6fbabf2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T17:13:29.218037106Z\",\n \"endTime\": \"2026-09-16T17:37:32.760378539Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7j1gufforhdfccicslrrj2jqu4vgo still exists (deletion dispatched at 2026-09-16T17:13:30Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0010ae0d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-gwq76w\" in resourcegroup=\"rg-zstream-upgrade-4-21-kkt25v682kzc\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a1f1fe55-fa55-489a-97de-ae14b6fbabf2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a1f1fe55-fa55-489a-97de-ae14b6fbabf2\",\n \"name\": \"a1f1fe55-fa55-489a-97de-ae14b6fbabf2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T17:13:29.218037106Z\",\n \"endTime\": \"2026-09-16T17:37:32.760378539Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7j1gufforhdfccicslrrj2jqu4vgo still exists (deletion dispatched at 2026-09-16T17:13:30Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9fd33bab139e4a398e3a990c65cceb30b097c0dd077e1c50321b830e3cae85e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","failed_at":"alert","signature_id":"fde97b6bc2d22cd4f0aa0ec9116922422f6896f61d1b4b08310d88dfdf610931","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 6 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T17:20:46Z\n Ended: 2026-09-16T17:40:45Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4807552-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-5fjfs\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-5fjfs (namespace arobit) on cluster ci01-j4807552-svc has averaged 184.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T17:20:46Z\n Ended: 2026-09-16T17:46:47Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4807552-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-86c59d8c49-4xp9s\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-86c59d8c49-4xp9s (namespace clusters-service) on cluster ci01-j4807552-svc has averaged 237.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T17:20:47Z\n Ended: 2026-09-16T17:33:46Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4807552-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-fzgp4\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-fzgp4 (namespace arobit) on cluster ci01-j4807552-mgmt-2 has averaged 206.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-16T17:20:47Z\n Ended: 2026-09-16T17:33:47Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4807552-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-8545674775-t5cps\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-8545674775-t5cps (namespace mgmt-agent) on cluster ci01-j4807552-mgmt-2 has averaged 223.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-16T17:20:47Z\n Ended: 2026-09-16T17:41:46Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4807552-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-86c59d8c49-hskmf\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-86c59d8c49-hskmf (namespace clusters-service) on cluster ci01-j4807552-svc has averaged 213.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Resolved\n Started: 2026-09-16T17:20:47Z\n Ended: 2026-09-16T17:54:45Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4807552-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-64d79bb67-smtrr\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-64d79bb67-smtrr (namespace aro-hcp) on cluster ci01-j4807552-svc has averaged 430.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"a069ca49223543cd11950aa682afe3f7690771360bf1072550bce45d5925534b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","failed_at":"alert","signature_id":"ed2b24c463c24323e023a9ff56bf6535b99fca0fd95a6e6a5f3b0c1f68b41036","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T17:54:24Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j4807552-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.89:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4807552/providers/microsoft.monitor/accounts/services-j4807552\", namespace=\"klusterlet-2ss7is141l1pe3gtmk90b2cej7ltp3gn\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-zvqnn\", prometheus=\"prometheus/prometheus\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2ss7is141l1pe3gtmk90b2cej7ltp3gn on cluster ci01-j4807552-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-16T17:54:24Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j4807552-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.89:8080\", job=\"kube-state-metrics\", lease=\"config-policy-controller\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4807552/providers/microsoft.monitor/accounts/services-j4807552\", namespace=\"klusterlet-2ss7is141l1pe3gtmk90b2cej7ltp3gn\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-zvqnn\", prometheus=\"prometheus/prometheus\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease config-policy-controller in namespace klusterlet-2ss7is141l1pe3gtmk90b2cej7ltp3gn on cluster ci01-j4807552-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-16T17:54:24Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j4807552-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.89:8080\", job=\"kube-state-metrics\", lease=\"governance-policy-framework\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4807552/providers/microsoft.monitor/accounts/services-j4807552\", namespace=\"klusterlet-2ss7is141l1pe3gtmk90b2cej7ltp3gn\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-zvqnn\", prometheus=\"prometheus/prometheus\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease governance-policy-framework in namespace klusterlet-2ss7is141l1pe3gtmk90b2cej7ltp3gn on cluster ci01-j4807552-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}},{"row_id":"a45d33821e9094ef3535365dabe4ee751d254bff06ff7572fcae5700114b66bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","failed_at":"e2e","signature_id":"460210ebabcf024c04114b174b6545f46b58b1c28d70303d86e3b1dce093c204","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001af2090\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-ffprrc6mwxtw\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8a335987-cce9-48c8-b521-939bf1363ace\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8a335987-cce9-48c8-b521-939bf1363ace\",\n \"name\": \"8a335987-cce9-48c8-b521-939bf1363ace\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T16:57:55.525213606Z\",\n \"endTime\": \"2026-09-16T17:22:02.784429606Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7j2du92su3qg8tbr84a8d1euhi38t still exists (deletion dispatched at 2026-09-16T16:57:56Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001af2090\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-ffprrc6mwxtw\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8a335987-cce9-48c8-b521-939bf1363ace\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8a335987-cce9-48c8-b521-939bf1363ace\",\n \"name\": \"8a335987-cce9-48c8-b521-939bf1363ace\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T16:57:55.525213606Z\",\n \"endTime\": \"2026-09-16T17:22:02.784429606Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7j2du92su3qg8tbr84a8d1euhi38t still exists (deletion dispatched at 2026-09-16T16:57:56Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"aef0a5a9544f3c578b00e47bc99665120e63dfdaaae0bf2013c4c1bbfef950e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","failed_at":"e2e","signature_id":"4f20aa230f0afb2f03a0196f22a9def6c79b679a7bc9bc79347900fd06c241a8","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0014240f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-xbmtbzvlpclx\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d85e799c-786e-4172-95d1-86693923282f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d85e799c-786e-4172-95d1-86693923282f\",\n \"name\": \"d85e799c-786e-4172-95d1-86693923282f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T17:32:57.751739191Z\",\n \"endTime\": \"2026-09-16T17:57:03.956555792Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7j51n0bvbqmvanc2bru5bjg1kuddr still exists (deletion dispatched at 2026-09-16T17:32:58Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0014240f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-xbmtbzvlpclx\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d85e799c-786e-4172-95d1-86693923282f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d85e799c-786e-4172-95d1-86693923282f\",\n \"name\": \"d85e799c-786e-4172-95d1-86693923282f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T17:32:57.751739191Z\",\n \"endTime\": \"2026-09-16T17:57:03.956555792Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7j51n0bvbqmvanc2bru5bjg1kuddr still exists (deletion dispatched at 2026-09-16T17:32:58Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"63a24fc72ea6be67478ba57ceca4f83bfef60aaeae2826b2abf0c75f70bec403","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b8c0f9b93a2229c5aa311b479a5bd95e0d09a73b0841f726046baada43f67050","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","failed_at":"e2e","signature_id":"f38603c674107979a7e5113b087deeb85d932689f0a71354d02c3879c176e60d","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001ab4060\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-22-8c8z82\" in resourcegroup=\"rg-zstream-upgrade-4-22-kjwnx2cdrdkq\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02850310-ff4f-4e8b-b9be-3ff8f6b73a30\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02850310-ff4f-4e8b-b9be-3ff8f6b73a30\",\n \"name\": \"02850310-ff4f-4e8b-b9be-3ff8f6b73a30\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T17:13:39.552363722Z\",\n \"endTime\": \"2026-09-16T17:37:42.765703073Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7iva927ialh1g5ofg1r8t781qfmn7 still exists (deletion dispatched at 2026-09-16T17:13:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001ab4060\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-22-8c8z82\" in resourcegroup=\"rg-zstream-upgrade-4-22-kjwnx2cdrdkq\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02850310-ff4f-4e8b-b9be-3ff8f6b73a30\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02850310-ff4f-4e8b-b9be-3ff8f6b73a30\",\n \"name\": \"02850310-ff4f-4e8b-b9be-3ff8f6b73a30\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T17:13:39.552363722Z\",\n \"endTime\": \"2026-09-16T17:37:42.765703073Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7iva927ialh1g5ofg1r8t781qfmn7 still exists (deletion dispatched at 2026-09-16T17:13:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ed0f24f494cb68e0cf6ea52e4f7db18a9f84bda8e210759c2dabbdaf0448a453","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","failed_at":"e2e","signature_id":"4b15da3ce1591e33d2696652650de2d0cdc6c841165c33a2a923881126078487","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000a9c270\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-fqmsfk\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-lqr7gtcm4srq\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f5ff4f5-7eef-4834-9b6c-c2e1109ad259\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f5ff4f5-7eef-4834-9b6c-c2e1109ad259\",\n \"name\": \"3f5ff4f5-7eef-4834-9b6c-c2e1109ad259\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T17:13:26.770029811Z\",\n \"endTime\": \"2026-09-16T17:37:32.768384818Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7j3fnpr9m9cqfjaohaiq73k6sjvpi still exists (deletion dispatched at 2026-09-16T17:13:27Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000a9c270\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-fqmsfk\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-lqr7gtcm4srq\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f5ff4f5-7eef-4834-9b6c-c2e1109ad259\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f5ff4f5-7eef-4834-9b6c-c2e1109ad259\",\n \"name\": \"3f5ff4f5-7eef-4834-9b6c-c2e1109ad259\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T17:13:26.770029811Z\",\n \"endTime\": \"2026-09-16T17:37:32.768384818Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7j3fnpr9m9cqfjaohaiq73k6sjvpi still exists (deletion dispatched at 2026-09-16T17:13:27Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"63a24fc72ea6be67478ba57ceca4f83bfef60aaeae2826b2abf0c75f70bec403","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":10,"failure_pattern_ids":["1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","63a24fc72ea6be67478ba57ceca4f83bfef60aaeae2826b2abf0c75f70bec403","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","b2d9bdf10bd756a41af895df7f16c38d9b8ca7729eae2184e9c4375fba7db5fb"],"matched_occurrences":10,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6982,"pr_state":"open","pr_sha":"74ce6804bb0c459935ed2ada3f3ac6939fdc47ea","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T15:42:06Z","started_at":"2026-09-16T15:42:06Z","completed_at":"2026-09-16T17:47:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:08.779838257Z","timing_metadata_checked_at":"2026-09-18T15:41:08.779838257Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:08.783458871Z","region_metadata_checked_at":"2026-09-18T15:41:08.783458871Z"},"failed_at":["alert","e2e"],"failed_test_count":10,"occurrences":[{"row_id":"1063ac505c4823fc7ba7492e00cecccce29171a320632d1f13fcfa477a8ddfde","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","failed_at":"alert","signature_id":"59897c5f85f7527efbbab27b1d42a8c63b62d2873c4723537141e6e4cb09a74c","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T16:36:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-cwxxr\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod kube-system/aks-secrets-store-csi-driver-cwxxr has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-16T16:36:14Z\n Ended: 2026-09-16T17:19:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-28181016-vmss000000-vxff5\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod kube-system/eraser-aks-system-28181016-vmss000000-vxff5 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-16T16:36:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-xfmjp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod kube-system/ama-metrics-node-xfmjp has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"253e7dc2148420208d3a32b153e63d827c991244bdfbb687a16333cbf5ba1b5c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","failed_at":"alert","signature_id":"314e25f4f6bda2d5748f9fe5aa89f88eaeb725849f7741eef5e0355c2394136c","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T17:11:51Z\n Ended: 2026-09-16T17:29:53Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/hcps-j6091136\", namespace=\"ocm-arohcpci01-2ss7f44euk2o4nqkrp28r72hngjs9t5m-g7f9d6m4t8e6w1m\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ss7f44euk2o4nqkrp28r72hngjs9t5m-g7f9d6m4t8e6w1m/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"6f30cbc178ad859302aa0e1e9a23830b28cd3fcd6a4c104d485caed99f7c3f38","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","failed_at":"alert","signature_id":"8cefc20a001466adc6a61bb43c8407f0170786f35f4348a8d83cb4fc19e10669","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T16:45:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/services-j6091136\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-16T16:45:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/services-j6091136\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-16T16:46:17Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/services-j6091136\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-16T16:46:17Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/services-j6091136\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired"}},{"row_id":"b9673f185b07c59df7a74caeb4f317285305c8b1b9aae5fcba02f270cbe5eb17","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","failed_at":"alert","signature_id":"97461ac292fa9fba61fa81d505ebf8a88d3f96ffd9a07f7d644245f3dd5aa26b","test_name":"[aro-hcp-observability] [svc] alert KubeContainerWaiting does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 5 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T17:30:16Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"secrets-store\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-cwxxr\", region=\"canadacentral\", severity=\"warning\"\n Description: pod/aks-secrets-store-csi-driver-cwxxr in namespace kube-system on container secrets-store has been in waiting state for longer than 1 hour.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-16T17:30:16Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"liveness-probe\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-cwxxr\", region=\"canadacentral\", severity=\"warning\"\n Description: pod/aks-secrets-store-csi-driver-cwxxr in namespace kube-system on container liveness-probe has been in waiting state for longer than 1 hour.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-16T17:30:16Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"node-driver-registrar\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-cwxxr\", region=\"canadacentral\", severity=\"warning\"\n Description: pod/aks-secrets-store-csi-driver-cwxxr in namespace kube-system on container node-driver-registrar has been in waiting state for longer than 1 hour.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-16T17:30:16Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"prometheus-collector\", namespace=\"kube-system\", pod=\"ama-metrics-node-xfmjp\", region=\"canadacentral\", severity=\"warning\"\n Description: pod/ama-metrics-node-xfmjp in namespace kube-system on container prometheus-collector has been in waiting state for longer than 1 hour.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-09-16T17:30:16Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"addon-token-adapter\", namespace=\"kube-system\", pod=\"ama-metrics-node-xfmjp\", region=\"canadacentral\", severity=\"warning\"\n Description: pod/ama-metrics-node-xfmjp in namespace kube-system on container addon-token-adapter has been in waiting state for longer than 1 hour.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a89f0d2edcd262580a6fd914503ced16ad52ae763f2c2255b4f5af55d375ecd3","failure_pattern":"alert [svc] KubeContainerWaiting fired","search_query":"alert [svc] KubeContainerWaiting fired"}},{"row_id":"bd66aa6aebcdaba1e9ad40b4d1c1fb868c1e3c51fa8030102cc76b641e9b264b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","failed_at":"alert","signature_id":"b995781a6685881ffdb2f7b53822eec544ca3973eb89f51c67b3781f51c35c18","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T16:56:51Z\n Ended: 2026-09-16T17:24:50Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/hcps-j6091136\", namespace=\"ocm-arohcpci01-2ss7f44euk2o4nqkrp28r72hngjs9t5m-g7f9d6m4t8e6w1m\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2ss7f44euk2o4nqkrp28r72hngjs9t5m-g7f9d6m4t8e6w1m/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"c2f0628121f85792c3f43d9cdb7e2aa052e750b4b2b1c922c393fdf3d12ec100","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","failed_at":"alert","signature_id":"e58f291d243075248b87935d2709ed85f1b36fdb31f78f88aaae185cc551feae","test_name":"[aro-hcp-observability] [svc] alert KubeNodeNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T17:00:36Z\n Severity: Sev3\n Labels: alertname=\"KubeNodeNotReady\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Ready\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/services-j6091136\", namespace=\"prometheus\", node=\"aks-system-28181016-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"true\"\n Description: aks-system-28181016-vmss000000 has been unready for more than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-16T17:00:36Z\n Severity: Sev3\n Labels: alertname=\"KubeNodeNotReady\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Ready\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/services-j6091136\", namespace=\"prometheus\", node=\"aks-system-28181016-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"true\"\n Description: aks-system-28181016-vmss000000 has been unready for more than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d6c6b690ca60846dd98e464313d6a9672b9abd8744604edd915627eb68203f9d","failure_pattern":"alert [svc] KubeNodeNotReady fired","search_query":"alert [svc] KubeNodeNotReady fired"}},{"row_id":"c51491862b2ca0ebf053cd561f32bf5a9fdcbc130bcab18a0cf67a78615b3587","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","failed_at":"alert","signature_id":"cd9f479af97dbe998875874266d9ea41c8e58990a5cf6ae60b1ca79fabc876a2","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T17:11:15Z\n Ended: 2026-09-16T17:17:14Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/services-j6091136\", namespace=\"klusterlet-2ss7f44euk2o4nqkrp28r72hngjs9t5m\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2ss7f44euk2o4nqkrp28r72hngjs9t5m/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}},{"row_id":"da85a1ba11468a5fcafa326b512c6aa236b64981661a23ca4e8866b201036547","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","failed_at":"alert","signature_id":"5176305f0eeec8bbbaa6ff316c0143a9416bf749550533697343b8a801ca28cd","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 6 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T17:11:36Z\n Ended: 2026-09-16T17:33:35Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6091136-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-8565d85cf5-nxd7f\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-8565d85cf5-nxd7f (namespace clusters-service) on cluster ci01-j6091136-svc has averaged 261.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T17:11:36Z\n Ended: 2026-09-16T17:21:35Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6091136-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-59b756d996-zw9bn\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-59b756d996-zw9bn (namespace mgmt-agent) on cluster ci01-j6091136-mgmt-1 has averaged 157.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T17:11:36Z\n Ended: 2026-09-16T17:19:36Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6091136-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-fjjcj\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-fjjcj (namespace arobit) on cluster ci01-j6091136-mgmt-1 has averaged 150.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-16T17:11:36Z\n Ended: 2026-09-16T17:29:36Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6091136-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-8565d85cf5-xznk2\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-8565d85cf5-xznk2 (namespace clusters-service) on cluster ci01-j6091136-svc has averaged 235.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-16T17:11:36Z\n Ended: 2026-09-16T17:29:37Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6091136-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-5b49bd5fc7-4mpg6\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-5b49bd5fc7-4mpg6 (namespace aro-hcp) on cluster ci01-j6091136-svc has averaged 438.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Resolved\n Started: 2026-09-16T17:11:36Z\n Ended: 2026-09-16T17:29:37Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6091136-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-gttgk\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-gttgk (namespace arobit) on cluster ci01-j6091136-svc has averaged 224.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"e35a2269b2f65b0d3faaeab73d57d2620e0e5c917436d975efbabaa7948c9bc6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","failed_at":"e2e","signature_id":"9ae40ec334e2759457083c312bb3ac6d6fef401546cf0c8751fc4d293a56e796","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b9a460\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-ph4gnm2bfn65\" to finish creating: GET https://rp.j6091136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ea12950c-66bd-4bf3-89b1-519a01fad530\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ea12950c-66bd-4bf3-89b1-519a01fad530\",\n \"name\": \"ea12950c-66bd-4bf3-89b1-519a01fad530\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T16:36:32.246966127Z\",\n \"endTime\": \"2026-09-16T17:05:32.533743336Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b9a460\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-ph4gnm2bfn65\" to finish creating: GET https://rp.j6091136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ea12950c-66bd-4bf3-89b1-519a01fad530\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ea12950c-66bd-4bf3-89b1-519a01fad530\",\n \"name\": \"ea12950c-66bd-4bf3-89b1-519a01fad530\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T16:36:32.246966127Z\",\n \"endTime\": \"2026-09-16T17:05:32.533743336Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f8bf2bd11f5c8236f1704e2237922765ecfec3c32909af9cbf2dc7494dd20d54","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","failed_at":"alert","signature_id":"4891bae57c16737882a8247821ec7f6d4e17417de53801a0bcb72589d84d269e","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T16:39:46Z\n Ended: 2026-09-16T17:12:43Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6091136-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6091136-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":10,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","a89f0d2edcd262580a6fd914503ced16ad52ae763f2c2255b4f5af55d375ecd3","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","d6c6b690ca60846dd98e464313d6a9672b9abd8744604edd915627eb68203f9d","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c"],"matched_occurrences":10,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6923/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100244030364323840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6923,"pr_state":"closed","pr_sha":"9e6650394c80bca242363572272841073766ea22","final_merged_sha":"9e6650394c80bca242363572272841073766ea22","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T15:22:38Z","started_at":"2026-09-16T15:22:38Z","completed_at":"2026-09-16T17:21:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:41.461378757Z","timing_metadata_checked_at":"2026-09-18T15:40:41.461378757Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:41.465350671Z","region_metadata_checked_at":"2026-09-18T15:40:41.465350671Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"14f049f82c8e7e43c3a8881a759aef8374e17377b8c86787861e6b254bbf38f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6923/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100244030364323840","occurred_at":"2026-09-16T15:22:38Z","failed_at":"alert","signature_id":"3af7515b2be64cbe2958e1955e72cb60cdb6b11e0681c1a634bbc8ed9f8e15d6","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T16:36:22Z\n Ended: 2026-09-16T16:43:22Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4323840-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-gctwd\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-gctwd (namespace arobit) on cluster ci01-j4323840-mgmt-2 has averaged 156.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100243112008880128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6923,"pr_state":"closed","pr_sha":"9e6650394c80bca242363572272841073766ea22","final_merged_sha":"9e6650394c80bca242363572272841073766ea22","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T15:19:04Z","started_at":"2026-09-16T15:19:04Z","completed_at":"2026-09-16T17:18:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:23.291048257Z","timing_metadata_checked_at":"2026-09-18T15:41:23.291048257Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:23.294135968Z","region_metadata_checked_at":"2026-09-18T15:41:23.294135968Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"104300c9e5d3b1bae0fe53136942c0e89380b2fc6ed7aa287c38c1774d5d2daa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100243112008880128","occurred_at":"2026-09-16T15:19:04Z","failed_at":"e2e","signature_id":"e0666d1ee60ed5c6d4f663533ea9b4f979d5b8183e061f975c9361ca0cafa025","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.010s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001cb64b0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.010s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001cb64b0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}},{"row_id":"3d2c4aad2bda2f32f210dc5ffa3e4ce6d823ec89c2bc78e4cbd0d80443e9bbe5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100243112008880128","occurred_at":"2026-09-16T15:19:04Z","failed_at":"alert","signature_id":"8cecf1b875c80d5aaad598063b3c383cfb7ef9d6225c392907d6632715156b7b","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T16:34:22Z\n Ended: 2026-09-16T16:42:25Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j8880128-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-856989784f-nd6qv\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-856989784f-nd6qv (namespace clusters-service) on cluster ci01-j8880128-svc has averaged 259.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6967/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100241269031702528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6967,"pr_state":"open","pr_sha":"9e548e6a88041aa01e2f2bfe95ccbcea3e5f3640","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-16T15:11:53Z","started_at":"2026-09-16T15:11:53Z","completed_at":"2026-09-16T17:06:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:00.729671168Z","timing_metadata_checked_at":"2026-09-18T15:41:00.729671168Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:00.732686779Z","region_metadata_checked_at":"2026-09-18T15:41:00.732686779Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100237146764873728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6752,"pr_state":"closed","pr_sha":"1cb34d14b09ab861f56c19f2a833972dafcaa9d9","final_merged_sha":"d433c21668cfdc66f98262f9099c8086b4397248","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T14:55:30Z","started_at":"2026-09-16T14:55:30Z","completed_at":"2026-09-16T16:54:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:19.844393488Z","timing_metadata_checked_at":"2026-09-18T15:40:19.844393488Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:19.849553506Z","region_metadata_checked_at":"2026-09-18T15:40:19.849553506Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"2126964f98873bc4fdc2d6fc9da00a06300b9be5a58f1a9c5ebcaaa7e399d2f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100237146764873728","occurred_at":"2026-09-16T14:55:30Z","failed_at":"e2e","signature_id":"6628b29b597aa4c3bb50125f87195e6a58208831ab9330e6078d3d1c523d7af7","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:144]: Admin credentials did not return 409 conflict error while cluster is deploying\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:144]: Admin credentials did not return 409 conflict error while cluster is deploying","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f59e236e067df212a87e97ef00e5cb14f86f6db93d160e4c5cd41734c06ea1b3","failure_pattern":"Admin credentials did not return 409 conflict error while cluster is deploying","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:144]: Admin credentials did not return 409 conflict error while cluster is deploying"}},{"row_id":"9829a693644934c9346faba8a4cea56c3196d468efb313788f0a547cda261a99","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100237146764873728","occurred_at":"2026-09-16T14:55:30Z","failed_at":"e2e","signature_id":"493948c9ce416bdc516799e228ff28ed99f8807ce0868ea1c0984eabe1af5cd2","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0018f0700\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0018f0700\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","f59e236e067df212a87e97ef00e5cb14f86f6db93d160e4c5cd41734c06ea1b3"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6985/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100236229235707904","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6985,"pr_state":"closed","pr_sha":"f309de9c65f1cf436f74ee89ec846b4d60ca9649","final_merged_sha":"d7b0b57bd15ef0bbc4edf6ad998bd65072d074ae","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-16T14:51:52Z","started_at":"2026-09-16T14:51:52Z","completed_at":"2026-09-16T17:05:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:09.346970907Z","timing_metadata_checked_at":"2026-09-18T15:41:09.346970907Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:09.350014118Z","region_metadata_checked_at":"2026-09-18T15:41:09.350014118Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100233505077202944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6980,"pr_state":"closed","pr_sha":"6231a1c9c8d45732bb81e9e2336cd167c45ee95d","final_merged_sha":"4c6ef86c4410d1e48dd7653c177a4370cf38a246","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T14:41:02Z","started_at":"2026-09-16T14:41:02Z","completed_at":"2026-09-16T15:17:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:07.28966299Z","timing_metadata_checked_at":"2026-09-18T15:41:07.28966299Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:07.292824201Z","region_metadata_checked_at":"2026-09-18T15:41:07.292824201Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"66a720ca0eed8bf225f1e378f7adc5f310a31944b032e36296260fcd88b7ec75","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100233505077202944","occurred_at":"2026-09-16T14:41:02Z","failed_at":"provision","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","test_suite":"step graph","failure_text":"time=2026-09-16T15:02:01.563Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-16T15:02:02.231Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-16T15:02:03.235Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=9b458a858b62ced8b861c50221db722ca7b84af5c0247e7866fc1bf068dbe8ac portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7202944-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9b458a858b62ced8b861c50221db722ca7b84af5c0247e7866fc1bf068dbe8ac\ntime=2026-09-16T15:11:05.045Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.Resources/deployments/9b458a858b62ced8b861c50221db722ca7b84af5c0247e7866fc1bf068dbe8ac/operationStatuses/08584120359627896142\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.Resources/deployments/cluster-oqca7h7ulqwlw\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7202944-mgmt-2/agentPools/userswft3\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 96, remaining 48 for family StandardEdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7202944-mgmt-2/agentPools/userswft2\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 96, remaining 48 for family StandardEdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7202944-mgmt-2/agentPools/userswft1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 96, remaining 48 for family StandardEdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ef6114015f72fab4d7dec149451d78cb6b761172f511b8fdedd7928043056bed","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ErrCode_InsufficientVCPUQuota; detail message Insufficient vcpu quota requested \u003ccount\u003e, remaining \u003ccount\u003e for family StandardEdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: \u003curl\u003e Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ef6114015f72fab4d7dec149451d78cb6b761172f511b8fdedd7928043056bed"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6984/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100232232680558592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6984,"pr_state":"closed","pr_sha":"9cd3a2f95079dcceb05fd8efa1a8b58c93f301a9","final_merged_sha":"9cd3a2f95079dcceb05fd8efa1a8b58c93f301a9","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-16T14:35:57Z","started_at":"2026-09-16T14:35:57Z","completed_at":"2026-09-16T16:38:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:09.04601712Z","timing_metadata_checked_at":"2026-09-18T15:41:09.04601712Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:09.053271846Z","region_metadata_checked_at":"2026-09-18T15:41:09.053271846Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6880,"pr_state":"closed","pr_sha":"8331ce339f8cc3291a75e15294e3fa6476aae41c","final_merged_sha":"a20dd280dd6b7499cf853051219487314ef1e460","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T14:31:25Z","started_at":"2026-09-16T14:31:25Z","completed_at":"2026-09-16T16:37:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:30.711639883Z","timing_metadata_checked_at":"2026-09-18T15:40:30.711639883Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:30.714581094Z","region_metadata_checked_at":"2026-09-18T15:40:30.714581094Z"},"failed_at":["alert","e2e"],"failed_test_count":8,"occurrences":[{"row_id":"2d0f1268a1bb486297a1c1380123b41a786d039f6cb2e9b7af38cac96ebed09b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","failed_at":"e2e","signature_id":"f0391e98a8d1c457ea51b54726e3c7af785fdf41b5e338fe271c88dbc805d911","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c1a440\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-6zdxmb6rpjzk\" to finish creating: GET https://rp.j3034240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/05315bbb-661b-4d11-978b-563d64e9ecf5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/05315bbb-661b-4d11-978b-563d64e9ecf5\",\n \"name\": \"05315bbb-661b-4d11-978b-563d64e9ecf5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:22:49.128060799Z\",\n \"endTime\": \"2026-09-16T15:51:54.335440454Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 20m47s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c1a440\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-6zdxmb6rpjzk\" to finish creating: GET https://rp.j3034240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/05315bbb-661b-4d11-978b-563d64e9ecf5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/05315bbb-661b-4d11-978b-563d64e9ecf5\",\n \"name\": \"05315bbb-661b-4d11-978b-563d64e9ecf5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:22:49.128060799Z\",\n \"endTime\": \"2026-09-16T15:51:54.335440454Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 20m47s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"94c048c28de1b18d33acb438d77867eeba7a4dce20d8722e1e707ad71dc67457","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","failed_at":"e2e","signature_id":"8553584f5882f3e24dad240bf62c9d51df1d078a5dc179dc15818be725a00b81","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a8ea00\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-67blpgs4n85p\" to finish creating: GET https://rp.j3034240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/41037149-7747-47d4-afb8-5a24484b3554\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/41037149-7747-47d4-afb8-5a24484b3554\",\n \"name\": \"41037149-7747-47d4-afb8-5a24484b3554\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:22:40.908389844Z\",\n \"endTime\": \"2026-09-16T15:51:44.402910752Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m28s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a8ea00\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-67blpgs4n85p\" to finish creating: GET https://rp.j3034240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/41037149-7747-47d4-afb8-5a24484b3554\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/41037149-7747-47d4-afb8-5a24484b3554\",\n \"name\": \"41037149-7747-47d4-afb8-5a24484b3554\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:22:40.908389844Z\",\n \"endTime\": \"2026-09-16T15:51:44.402910752Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m28s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bb95558a5a07a9be3649d6a3d061f121637260c0174c0d0e549144860d48a6f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","failed_at":"e2e","signature_id":"976862ec83bbad292378adfc6b6876d3447587a76e0f813a8827a52bd7db4bbc","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d7dc40\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-7vbqmtrz9vrj\" to finish creating: GET https://rp.j3034240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a28fe40a-0fe9-4492-945f-7a7bb07d9075\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a28fe40a-0fe9-4492-945f-7a7bb07d9075\",\n \"name\": \"a28fe40a-0fe9-4492-945f-7a7bb07d9075\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:24:46.641065736Z\",\n \"endTime\": \"2026-09-16T15:53:54.422419253Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d7dc40\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-7vbqmtrz9vrj\" to finish creating: GET https://rp.j3034240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a28fe40a-0fe9-4492-945f-7a7bb07d9075\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a28fe40a-0fe9-4492-945f-7a7bb07d9075\",\n \"name\": \"a28fe40a-0fe9-4492-945f-7a7bb07d9075\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:24:46.641065736Z\",\n \"endTime\": \"2026-09-16T15:53:54.422419253Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b414f524aec7c4ea3c6d464450b9af6f734740715a61aff1c3a8f53547f62641","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c816cb538b926cde61a131112a37c157b0336d23d08bd0c36fa324aac5286031","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","failed_at":"e2e","signature_id":"51205dcbc010a62ffa53522b05b81b83d3e49f95a801c1ca50c89cf5aa533563","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d66760\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-6s59tj5f5cwc\" to finish creating: GET https://rp.j3034240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf873d80-2639-4560-9d03-e297040bde4f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf873d80-2639-4560-9d03-e297040bde4f\",\n \"name\": \"bf873d80-2639-4560-9d03-e297040bde4f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:22:59.281142097Z\",\n \"endTime\": \"2026-09-16T15:52:04.370330727Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m17s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d66760\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-6s59tj5f5cwc\" to finish creating: GET https://rp.j3034240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf873d80-2639-4560-9d03-e297040bde4f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf873d80-2639-4560-9d03-e297040bde4f\",\n \"name\": \"bf873d80-2639-4560-9d03-e297040bde4f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:22:59.281142097Z\",\n \"endTime\": \"2026-09-16T15:52:04.370330727Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m17s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e3d30fee2e0523a124c746f184860be4124ff2f2a9a29098a8672670586b42c6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","failed_at":"e2e","signature_id":"9bdd91449d29da7a6a1036ded0dff71e7cee7dca6d3b72ee155f7272dbc7ec9e","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000efe540\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-62bnx4vg8k7q\" to finish creating: GET https://rp.j3034240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3ea6c341-7276-4abb-bbae-9fdf251b0d28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3ea6c341-7276-4abb-bbae-9fdf251b0d28\",\n \"name\": \"3ea6c341-7276-4abb-bbae-9fdf251b0d28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:22:30.840455999Z\",\n \"endTime\": \"2026-09-16T15:51:34.353957305Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m54s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000efe540\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-62bnx4vg8k7q\" to finish creating: GET https://rp.j3034240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3ea6c341-7276-4abb-bbae-9fdf251b0d28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3ea6c341-7276-4abb-bbae-9fdf251b0d28\",\n \"name\": \"3ea6c341-7276-4abb-bbae-9fdf251b0d28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:22:30.840455999Z\",\n \"endTime\": \"2026-09-16T15:51:34.353957305Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m54s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e49e3809cf361c29fe798ac46e2425a945d748273d98ab9afa6f8f3b10b5e429","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","failed_at":"e2e","signature_id":"7b40ea74a666a71f36b6120544e0bab8b70aaea87e5e9b851b3d3cee913ad9be","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:98]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d32180\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-bvtz5scvbwbv\" to finish creating: GET https://rp.j3034240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d32da0c1-6650-42fe-a236-8452aa971c7a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d32da0c1-6650-42fe-a236-8452aa971c7a\",\n \"name\": \"d32da0c1-6650-42fe-a236-8452aa971c7a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:22:40.242608551Z\",\n \"endTime\": \"2026-09-16T15:51:44.326484397Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m4s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:98]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d32180\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-bvtz5scvbwbv\" to finish creating: GET https://rp.j3034240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d32da0c1-6650-42fe-a236-8452aa971c7a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d32da0c1-6650-42fe-a236-8452aa971c7a\",\n \"name\": \"d32da0c1-6650-42fe-a236-8452aa971c7a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:22:40.242608551Z\",\n \"endTime\": \"2026-09-16T15:51:44.326484397Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m4s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f82791a04311fd8adbb7f8f022789d6d0bf48acedb4676c3aaa81bcc6b9adc3c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","failed_at":"alert","signature_id":"8a55cdb17b2e0960799afe76d087110a5ecf992561f62b085684e8010de63d22","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T15:37:33Z\n Ended: 2026-09-16T16:04:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3034240-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss6ciiqtdo86qknpl3sl69pgldega5p-a3j0h6i8k2j6q8r\", pod=\"router-64fb7b56c-wnsxq\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss6ciiqtdo86qknpl3sl69pgldega5p-a3j0h6i8k2j6q8r/router-64fb7b56c-wnsxq has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"fd38d1b564c1cc2d91e945f4abe365bed006d5319803aae7c116d5aff11daea6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","failed_at":"e2e","signature_id":"5f788a7a822406c498eab147df2fe3fad0c3f67d39005f710714ed9d67ac601d","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e4c880\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7vfjq4g5mpkz\" to finish creating: GET https://rp.j3034240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa4772cb-eb8a-477d-b8e0-5d1f516c5a63\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa4772cb-eb8a-477d-b8e0-5d1f516c5a63\",\n \"name\": \"fa4772cb-eb8a-477d-b8e0-5d1f516c5a63\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:25:02.43485078Z\",\n \"endTime\": \"2026-09-16T15:54:04.537444594Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e4c880\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7vfjq4g5mpkz\" to finish creating: GET https://rp.j3034240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa4772cb-eb8a-477d-b8e0-5d1f516c5a63\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa4772cb-eb8a-477d-b8e0-5d1f516c5a63\",\n \"name\": \"fa4772cb-eb8a-477d-b8e0-5d1f516c5a63\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:25:02.43485078Z\",\n \"endTime\": \"2026-09-16T15:54:04.537444594Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b414f524aec7c4ea3c6d464450b9af6f734740715a61aff1c3a8f53547f62641","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","b414f524aec7c4ea3c6d464450b9af6f734740715a61aff1c3a8f53547f62641"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"b368522ea717841a0bf803ebbe3e8dc7455b6e5e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T14:26:41Z","started_at":"2026-09-16T14:26:41Z","completed_at":"2026-09-16T16:40:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:05.356703948Z","timing_metadata_checked_at":"2026-09-18T15:40:05.356703948Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:05.359787259Z","region_metadata_checked_at":"2026-09-18T15:40:05.359787259Z"},"failed_at":["alert","e2e"],"failed_test_count":27,"occurrences":[{"row_id":"020032d2293b1de3eeee23008fff0a35d36980a3a70d5af3583f4efbcb16032b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","failed_at":"alert","signature_id":"3b4c127aae1159c832d5991591f49782774b64887255f529b831eb0acc43aae1","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T15:50:51Z\n Ended: 2026-09-16T15:58:54Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7769984-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.76:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7769984/providers/microsoft.monitor/accounts/services-j7769984\", namespace=\"klusterlet-2ss6631b7kcrqufnkep1rdniihr6hk1s\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-n4qlf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2ss6631b7kcrqufnkep1rdniihr6hk1s/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}},{"row_id":"092af1d79b557926301519c575c37f5c729330a4b280b157aa639b3af98f1049","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","failed_at":"e2e","signature_id":"daf2b2ff378776852d48a0372497ce4c540ea79fca95c306e900dac48324c4cd","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:444]: failed to create HCP cluster np-skip-minor-xf4ktl at version 4.22.14\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d984e0\u003e: \n failed to create HCP cluster np-skip-minor-xf4ktl: failed waiting for cluster=\"np-skip-minor-xf4ktl\" in resourcegroup=\"rg-np-skip-minor-xf4ktl-ndhlhsxtb9bp\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d9ddbedd-dd2c-4969-be11-096f362f19a9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d9ddbedd-dd2c-4969-be11-096f362f19a9\",\n \"name\": \"d9ddbedd-dd2c-4969-be11-096f362f19a9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:08:55.657642282Z\",\n \"endTime\": \"2026-09-16T15:38:00.339938436Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:444]: failed to create HCP cluster np-skip-minor-xf4ktl at version 4.22.14\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d984e0\u003e: \n failed to create HCP cluster np-skip-minor-xf4ktl: failed waiting for cluster=\"np-skip-minor-xf4ktl\" in resourcegroup=\"rg-np-skip-minor-xf4ktl-ndhlhsxtb9bp\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d9ddbedd-dd2c-4969-be11-096f362f19a9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d9ddbedd-dd2c-4969-be11-096f362f19a9\",\n \"name\": \"d9ddbedd-dd2c-4969-be11-096f362f19a9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:08:55.657642282Z\",\n \"endTime\": \"2026-09-16T15:38:00.339938436Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0dae61f5de650258fb1c9f7919ebcd9b8272c454872ba9b80bcaf319f9a8b4b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","failed_at":"e2e","signature_id":"4baa98b743ff58863c2c3f11ae6c7794d41c41f07100fe61766b0978f6746f19","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc001dd40f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2dq2jrpmm79t\" to finish deleting: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e014f5fe-d804-4d74-8be2-a041ccc5139a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e014f5fe-d804-4d74-8be2-a041ccc5139a\",\n \"name\": \"e014f5fe-d804-4d74-8be2-a041ccc5139a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:38:30.175216442Z\",\n \"endTime\": \"2026-09-16T16:02:31.265925549Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss65h8jgj0ruglgg8jptfh3v6s7qujv still exists (deletion dispatched at 2026-09-16T15:38:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc001dd40f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2dq2jrpmm79t\" to finish deleting: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e014f5fe-d804-4d74-8be2-a041ccc5139a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e014f5fe-d804-4d74-8be2-a041ccc5139a\",\n \"name\": \"e014f5fe-d804-4d74-8be2-a041ccc5139a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:38:30.175216442Z\",\n \"endTime\": \"2026-09-16T16:02:31.265925549Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss65h8jgj0ruglgg8jptfh3v6s7qujv still exists (deletion dispatched at 2026-09-16T15:38:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"129434ab01fdfa33111948e435e48518920dee77dad15941e667c800efa2122e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","failed_at":"e2e","signature_id":"0bf39d4aab091ac51c471f05a7fa67a4d591d3eb93555a71c26170936f523c5f","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:103]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001f3c120\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-sxmn5zhcsvjn\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a019abb2-ae16-4218-9707-6f7dc73a3eb3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a019abb2-ae16-4218-9707-6f7dc73a3eb3\",\n \"name\": \"a019abb2-ae16-4218-9707-6f7dc73a3eb3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:08:36.527287031Z\",\n \"endTime\": \"2026-09-16T15:37:40.20476418Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:103]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001f3c120\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-sxmn5zhcsvjn\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a019abb2-ae16-4218-9707-6f7dc73a3eb3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a019abb2-ae16-4218-9707-6f7dc73a3eb3\",\n \"name\": \"a019abb2-ae16-4218-9707-6f7dc73a3eb3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:08:36.527287031Z\",\n \"endTime\": \"2026-09-16T15:37:40.20476418Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fe6b01560ac49646fd0a46c48f672cf2fc0ebbf844ea4576bb0cab7a46df7ce8","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"151bb5b2ef7ce912b60275abba8863625a1fc79e52e48314880ff3036e80e366","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","failed_at":"e2e","signature_id":"52a3e55e5accf233dc0c4dcc31ad7c4bd1abab3ca6c7016627f681951931597f","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:93]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e96840\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-222ntj7gxqx2\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/774c9323-3626-4141-aa8c-a5f0d7e57e9c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/774c9323-3626-4141-aa8c-a5f0d7e57e9c\",\n \"name\": \"774c9323-3626-4141-aa8c-a5f0d7e57e9c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:07:38.983252228Z\",\n \"endTime\": \"2026-09-16T15:36:40.328092646Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:93]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e96840\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-222ntj7gxqx2\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/774c9323-3626-4141-aa8c-a5f0d7e57e9c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/774c9323-3626-4141-aa8c-a5f0d7e57e9c\",\n \"name\": \"774c9323-3626-4141-aa8c-a5f0d7e57e9c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:07:38.983252228Z\",\n \"endTime\": \"2026-09-16T15:36:40.328092646Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1e1c44d43176f96b660f85dfd5e6c16aa951bc184f76901219755fdef67d3436","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","failed_at":"e2e","signature_id":"eb1186f89f1efbd9f2fb8485a07c81a2445ab57e2c0781fc90a1717593153377","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00105cce0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-l5lpwk8c788b\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75be2192-167a-440e-8345-e0e86c88eab4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75be2192-167a-440e-8345-e0e86c88eab4\",\n \"name\": \"75be2192-167a-440e-8345-e0e86c88eab4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:07:49.433866246Z\",\n \"endTime\": \"2026-09-16T15:36:50.290217783Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m24s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00105cce0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-l5lpwk8c788b\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75be2192-167a-440e-8345-e0e86c88eab4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75be2192-167a-440e-8345-e0e86c88eab4\",\n \"name\": \"75be2192-167a-440e-8345-e0e86c88eab4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:07:49.433866246Z\",\n \"endTime\": \"2026-09-16T15:36:50.290217783Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m24s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"249a90ad652879100eef208b1cc4cf942226fdbc728e27c7cae19c768a50995f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","failed_at":"e2e","signature_id":"1081dc1f7ba277c79eab9706dd413878fd827376a9850ff6a2e23d1b663c5155","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:83]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000861120\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-pkbzl9cwqzmf\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7bc0c172-a0d6-40dc-821f-14c51cede823\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7bc0c172-a0d6-40dc-821f-14c51cede823\",\n \"name\": \"7bc0c172-a0d6-40dc-821f-14c51cede823\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:07:59.260634412Z\",\n \"endTime\": \"2026-09-16T15:37:00.334559736Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:83]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000861120\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-pkbzl9cwqzmf\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7bc0c172-a0d6-40dc-821f-14c51cede823\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7bc0c172-a0d6-40dc-821f-14c51cede823\",\n \"name\": \"7bc0c172-a0d6-40dc-821f-14c51cede823\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:07:59.260634412Z\",\n \"endTime\": \"2026-09-16T15:37:00.334559736Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"30fe1ccf1afb901847736bcb0fe5071b210284a48b76133fed80d8e095941b80","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","failed_at":"e2e","signature_id":"8eb85a382d8ce34de42657ea34ddc180803991e921b8e70810a099fe550ece42","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:112]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000baef60\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-r66n4dfd4blw\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5e8da50e-214e-46a9-b300-dcbc9025d457\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5e8da50e-214e-46a9-b300-dcbc9025d457\",\n \"name\": \"5e8da50e-214e-46a9-b300-dcbc9025d457\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:08:22.677421205Z\",\n \"endTime\": \"2026-09-16T15:37:30.284421662Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:112]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000baef60\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-r66n4dfd4blw\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5e8da50e-214e-46a9-b300-dcbc9025d457\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5e8da50e-214e-46a9-b300-dcbc9025d457\",\n \"name\": \"5e8da50e-214e-46a9-b300-dcbc9025d457\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:08:22.677421205Z\",\n \"endTime\": \"2026-09-16T15:37:30.284421662Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"38a553f8c28f9d34073b8d5fe52a266f02c9e8ed8fad188289696526dc01ff26","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","failed_at":"e2e","signature_id":"b323b711e4f15f03edb6542a780e728151255e7713e3d463bc33d5461bb8ccbf","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:420]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e08340\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-rjd9q4sr4pnl\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de1336a6-9373-4313-bfd1-74e55afef59f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de1336a6-9373-4313-bfd1-74e55afef59f\",\n \"name\": \"de1336a6-9373-4313-bfd1-74e55afef59f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:08:25.479355581Z\",\n \"endTime\": \"2026-09-16T15:37:30.356691506Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:420]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e08340\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-rjd9q4sr4pnl\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de1336a6-9373-4313-bfd1-74e55afef59f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de1336a6-9373-4313-bfd1-74e55afef59f\",\n \"name\": \"de1336a6-9373-4313-bfd1-74e55afef59f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:08:25.479355581Z\",\n \"endTime\": \"2026-09-16T15:37:30.356691506Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"42f35205985c289dc38947c1e1e830be1ccc35aabee5182fe335cfa55c55925d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","failed_at":"e2e","signature_id":"b2b0b3c510ba3d2ec5bcda27a848edbf0f6e4130232ba677304593697d08e0b2","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d8e640\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-mldclqf9sr5h\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/704efcc4-4b42-4704-9ff8-f01712701a44\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/704efcc4-4b42-4704-9ff8-f01712701a44\",\n \"name\": \"704efcc4-4b42-4704-9ff8-f01712701a44\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:09:00.206878787Z\",\n \"endTime\": \"2026-09-16T15:38:00.28474108Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d8e640\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-mldclqf9sr5h\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/704efcc4-4b42-4704-9ff8-f01712701a44\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/704efcc4-4b42-4704-9ff8-f01712701a44\",\n \"name\": \"704efcc4-4b42-4704-9ff8-f01712701a44\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:09:00.206878787Z\",\n \"endTime\": \"2026-09-16T15:38:00.28474108Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fe6b01560ac49646fd0a46c48f672cf2fc0ebbf844ea4576bb0cab7a46df7ce8","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"43d454b59c1a000be32a1b350e24745939e08ae96da28bf43d3b5915387ab1b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","failed_at":"e2e","signature_id":"cf0c7c2f7324da55d7b02b3480eb3ca7ec6d10efdd79ac18d6876f5a4e4fc8b8","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc000e90390\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-lqz4p67hcxjl\" to finish deleting: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ebdfc80f-b540-48be-aab0-8a738a247fdb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ebdfc80f-b540-48be-aab0-8a738a247fdb\",\n \"name\": \"ebdfc80f-b540-48be-aab0-8a738a247fdb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:36:32.444402833Z\",\n \"endTime\": \"2026-09-16T16:00:41.296180241Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss6623gm9lp6mh9kjc1ql22lej4dllq still exists (deletion dispatched at 2026-09-16T15:36:33Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc000e90390\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-lqz4p67hcxjl\" to finish deleting: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ebdfc80f-b540-48be-aab0-8a738a247fdb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ebdfc80f-b540-48be-aab0-8a738a247fdb\",\n \"name\": \"ebdfc80f-b540-48be-aab0-8a738a247fdb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:36:32.444402833Z\",\n \"endTime\": \"2026-09-16T16:00:41.296180241Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss6623gm9lp6mh9kjc1ql22lej4dllq still exists (deletion dispatched at 2026-09-16T15:36:33Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"479dd0298370cb6e9f48e808a9f24c146b4ef966b90abc171eed99285c37af30","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","failed_at":"e2e","signature_id":"6b6d7951973760ce40ada8345eab0fc3affb480bfb3af182b9b92a268d16b16b","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc0013de078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-7jq9hm2md6t8\" to finish deleting: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cf6b5b4a-2ab2-4c6d-916c-70bbc5bfcd3c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cf6b5b4a-2ab2-4c6d-916c-70bbc5bfcd3c\",\n \"name\": \"cf6b5b4a-2ab2-4c6d-916c-70bbc5bfcd3c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:35:11.305500228Z\",\n \"endTime\": \"2026-09-16T15:59:11.284085711Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss6632a8medpq72h75idr9hq6fk0kqj still exists (deletion dispatched at 2026-09-16T15:35:13Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc0013de078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-7jq9hm2md6t8\" to finish deleting: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cf6b5b4a-2ab2-4c6d-916c-70bbc5bfcd3c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cf6b5b4a-2ab2-4c6d-916c-70bbc5bfcd3c\",\n \"name\": \"cf6b5b4a-2ab2-4c6d-916c-70bbc5bfcd3c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:35:11.305500228Z\",\n \"endTime\": \"2026-09-16T15:59:11.284085711Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss6632a8medpq72h75idr9hq6fk0kqj still exists (deletion dispatched at 2026-09-16T15:35:13Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5e6cee6aa854caac9f4793715a3581e7ee1a84f6f82adfa283970850ec2fc0d8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","failed_at":"e2e","signature_id":"51fb4ad7b246c62ce6ea07724802f78706075364625cc9b0dc59f38a98c159cf","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:206]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002a80740\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-czltd5xxrlvj\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/180c48f3-76f8-4d48-bcf4-3b4e29dffab7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/180c48f3-76f8-4d48-bcf4-3b4e29dffab7\",\n \"name\": \"180c48f3-76f8-4d48-bcf4-3b4e29dffab7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:10:24.910079282Z\",\n \"endTime\": \"2026-09-16T15:39:30.319726782Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:206]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002a80740\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-czltd5xxrlvj\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/180c48f3-76f8-4d48-bcf4-3b4e29dffab7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/180c48f3-76f8-4d48-bcf4-3b4e29dffab7\",\n \"name\": \"180c48f3-76f8-4d48-bcf4-3b4e29dffab7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:10:24.910079282Z\",\n \"endTime\": \"2026-09-16T15:39:30.319726782Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5f9ceb603b2bb1f286a3a7de177fcd1915bfa446b1f93dbc16560d65bc7edac5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","failed_at":"e2e","signature_id":"92ab9e016befa8294f948260d25c72e7a1d80889506782e1f2a4bb4fd849697e","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:85]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b250e0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-rc8rlbszt67v\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/954e120a-f424-45bb-8dc4-e1e41d681776\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/954e120a-f424-45bb-8dc4-e1e41d681776\",\n \"name\": \"954e120a-f424-45bb-8dc4-e1e41d681776\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:07:38.834280426Z\",\n \"endTime\": \"2026-09-16T15:36:40.350051467Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m14s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:85]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b250e0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-rc8rlbszt67v\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/954e120a-f424-45bb-8dc4-e1e41d681776\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/954e120a-f424-45bb-8dc4-e1e41d681776\",\n \"name\": \"954e120a-f424-45bb-8dc4-e1e41d681776\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:07:38.834280426Z\",\n \"endTime\": \"2026-09-16T15:36:40.350051467Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m14s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"762e6c5678aeca07fc0ee0a7401589965bcc6627a41dc6d097c2f1642958dd8f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:183]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:183]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"7f59587487f6ffa3be25641dce025a1e2de3869f9f35b14f5aa48bfa3f33ba90","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","failed_at":"e2e","signature_id":"de1bc95acf9d163b0a4c5d87ba6c29060b3bb4b653bb97edd817c695a7ef262b","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:89]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f48160\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-hdp82z6x6stg\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/57c282c3-8a3f-435c-8e84-cc6479b9b24d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/57c282c3-8a3f-435c-8e84-cc6479b9b24d\",\n \"name\": \"57c282c3-8a3f-435c-8e84-cc6479b9b24d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:07:40.381050368Z\",\n \"endTime\": \"2026-09-16T15:36:40.345057819Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 14m43s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:89]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f48160\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-hdp82z6x6stg\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/57c282c3-8a3f-435c-8e84-cc6479b9b24d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/57c282c3-8a3f-435c-8e84-cc6479b9b24d\",\n \"name\": \"57c282c3-8a3f-435c-8e84-cc6479b9b24d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:07:40.381050368Z\",\n \"endTime\": \"2026-09-16T15:36:40.345057819Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 14m43s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"93fe0e17367a79e29305792cb42d179890a1232e104aaccb5d7a5e124c624a61","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","failed_at":"e2e","signature_id":"e41f400f85ccdcde13c1223820f617791e8453915fe2b4d9d6f910f58c9aac3e","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d24800\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-t46rl8c755dd\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a93fc974-cd28-42a1-85bb-63ca024b7fae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a93fc974-cd28-42a1-85bb-63ca024b7fae\",\n \"name\": \"a93fc974-cd28-42a1-85bb-63ca024b7fae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:08:54.45273143Z\",\n \"endTime\": \"2026-09-16T15:38:00.30841047Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d24800\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-t46rl8c755dd\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a93fc974-cd28-42a1-85bb-63ca024b7fae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a93fc974-cd28-42a1-85bb-63ca024b7fae\",\n \"name\": \"a93fc974-cd28-42a1-85bb-63ca024b7fae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:08:54.45273143Z\",\n \"endTime\": \"2026-09-16T15:38:00.30841047Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fe6b01560ac49646fd0a46c48f672cf2fc0ebbf844ea4576bb0cab7a46df7ce8","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9eabd69aa924e71e6b57654ea628a0c3f6f3ba61e1fe0e3a59295390b06cc85f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","failed_at":"e2e","signature_id":"53f4dbcf0527ee85fd51e8c2f839babdd74e956b703ba4e02f72dc66ca32e8b9","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:124]: failed to create HCP cluster \"cluster-zstream-5-0-ltfq8d\" with version 5.0.0-rc.1 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00019fbc0\u003e: \n failed to create HCP cluster cluster-zstream-5-0-ltfq8d: failed waiting for cluster=\"cluster-zstream-5-0-ltfq8d\" in resourcegroup=\"rg-zstream-upgrade-5-0-b9vtt7qlr7t6\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c61b5bd2-2bcc-415d-a1a0-e9dd864a49f0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c61b5bd2-2bcc-415d-a1a0-e9dd864a49f0\",\n \"name\": \"c61b5bd2-2bcc-415d-a1a0-e9dd864a49f0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:07:39.815926407Z\",\n \"endTime\": \"2026-09-16T15:36:40.332553644Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.1 is Partial (want Completed), started 16m33s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:124]: failed to create HCP cluster \"cluster-zstream-5-0-ltfq8d\" with version 5.0.0-rc.1 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00019fbc0\u003e: \n failed to create HCP cluster cluster-zstream-5-0-ltfq8d: failed waiting for cluster=\"cluster-zstream-5-0-ltfq8d\" in resourcegroup=\"rg-zstream-upgrade-5-0-b9vtt7qlr7t6\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c61b5bd2-2bcc-415d-a1a0-e9dd864a49f0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c61b5bd2-2bcc-415d-a1a0-e9dd864a49f0\",\n \"name\": \"c61b5bd2-2bcc-415d-a1a0-e9dd864a49f0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:07:39.815926407Z\",\n \"endTime\": \"2026-09-16T15:36:40.332553644Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.1 is Partial (want Completed), started 16m33s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ac01d97b1ddf3a478d9aaae12b8a775736e179b2b6202c8400262e6b109bee81","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","failed_at":"e2e","signature_id":"a64a06ed18c7d9e35b003b4339972fe04157c007a8cd68d29875724bb6a184d4","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--7tj5b6r9d28k/cluster-hs-pre-rwztb2 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ee48a0\u003e: \n failed to create HCP cluster cluster-hs-pre-rwztb2: failed waiting for cluster=\"cluster-hs-pre-rwztb2\" in resourcegroup=\"rg-hs-pre--7tj5b6r9d28k\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0015f221-0420-4231-b96b-76ce90032f9d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0015f221-0420-4231-b96b-76ce90032f9d\",\n \"name\": \"0015f221-0420-4231-b96b-76ce90032f9d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:07:38.092292569Z\",\n \"endTime\": \"2026-09-16T15:36:40.334174064Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--7tj5b6r9d28k/cluster-hs-pre-rwztb2 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ee48a0\u003e: \n failed to create HCP cluster cluster-hs-pre-rwztb2: failed waiting for cluster=\"cluster-hs-pre-rwztb2\" in resourcegroup=\"rg-hs-pre--7tj5b6r9d28k\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0015f221-0420-4231-b96b-76ce90032f9d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0015f221-0420-4231-b96b-76ce90032f9d\",\n \"name\": \"0015f221-0420-4231-b96b-76ce90032f9d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:07:38.092292569Z\",\n \"endTime\": \"2026-09-16T15:36:40.334174064Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fe6b01560ac49646fd0a46c48f672cf2fc0ebbf844ea4576bb0cab7a46df7ce8","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b201395dc4ddc56d6d15be3853113d56e4eee3f60cd270a5246ba2a55ff6cf54","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","failed_at":"alert","signature_id":"d5ed3091019088eb341f9671128472e74176d896eca057722a5636069c20079f","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T16:30:00Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7769984-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.76:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7769984/providers/microsoft.monitor/accounts/hcps-j7769984\", namespace=\"ocm-arohcpci01-2ss6631b7kcrqufnkep1rdniihr6hk1s-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-n4qlf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ss6631b7kcrqufnkep1rdniihr6hk1s-cilium-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-16T16:30:00Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7769984-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.76:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7769984/providers/microsoft.monitor/accounts/hcps-j7769984\", namespace=\"ocm-arohcpci01-2ss6631b7kcrqufnkep1rdniihr6hk1s-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-n4qlf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ss6631b7kcrqufnkep1rdniihr6hk1s-cilium-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"c1f2a4b45ccc5cb7ac2076e8f07aac599719cc2e832d8c142b57a412a436eb9e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","failed_at":"e2e","signature_id":"593263a65b95e6f760aa7210a803a4f621c8333076597839a59808dbbf66d1f5","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:111]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012ca380\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-t5bwqzxttz2x\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c7634186-3d3c-4d05-8ca0-3b5b6a83a2ab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c7634186-3d3c-4d05-8ca0-3b5b6a83a2ab\",\n \"name\": \"c7634186-3d3c-4d05-8ca0-3b5b6a83a2ab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:40:50.62183993Z\",\n \"endTime\": \"2026-09-16T16:09:59.134370699Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 25m16s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:111]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012ca380\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-t5bwqzxttz2x\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c7634186-3d3c-4d05-8ca0-3b5b6a83a2ab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c7634186-3d3c-4d05-8ca0-3b5b6a83a2ab\",\n \"name\": \"c7634186-3d3c-4d05-8ca0-3b5b6a83a2ab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:40:50.62183993Z\",\n \"endTime\": \"2026-09-16T16:09:59.134370699Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 25m16s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c971cc42609069696f473f4ad0e095dae446684c89e514ff155b3e634deb06c9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","failed_at":"e2e","signature_id":"09330ba162b6af089c34735c70eca9ddd91f6516659badb8a358d7c06fa7a1f1","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20260901preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc00228a378\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-rsrv4kf2w46v\" to finish deleting: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8e49de5-12cd-4e7e-9b28-73dc76953ac6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8e49de5-12cd-4e7e-9b28-73dc76953ac6\",\n \"name\": \"e8e49de5-12cd-4e7e-9b28-73dc76953ac6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:42:15.675788401Z\",\n \"endTime\": \"2026-09-16T16:06:21.284864271Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss6634t6efhs2ajqtn63hq5cv2cmsie still exists (deletion dispatched at 2026-09-16T15:42:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc00228a378\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-rsrv4kf2w46v\" to finish deleting: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8e49de5-12cd-4e7e-9b28-73dc76953ac6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8e49de5-12cd-4e7e-9b28-73dc76953ac6\",\n \"name\": \"e8e49de5-12cd-4e7e-9b28-73dc76953ac6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:42:15.675788401Z\",\n \"endTime\": \"2026-09-16T16:06:21.284864271Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss6634t6efhs2ajqtn63hq5cv2cmsie still exists (deletion dispatched at 2026-09-16T15:42:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cfc1ef4ad1c8c43e8ebeaf91b62509925192a9c2637e2370bf300af406704737","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","failed_at":"alert","signature_id":"d167cd058619d1445e1dc6c6ed93bc2e291c8bbbd9eba4b08df3c14423f77517","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 7 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T15:23:56Z\n Ended: 2026-09-16T15:50:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7769984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss65gp6p7cj2egifuj36635qckn05a0-p3q5k4a8x1k8d8w\", pod=\"router-6f94f5d6c4-ccjv2\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss65gp6p7cj2egifuj36635qckn05a0-p3q5k4a8x1k8d8w/router-6f94f5d6c4-ccjv2 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-16T15:24:56Z\n Ended: 2026-09-16T15:30:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7769984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss65mnnlogu9073uivmq2325bj8tnd2-u0g9g1i5r8s0n6y\", pod=\"router-854fb99ff8-k2fd5\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss65mnnlogu9073uivmq2325bj8tnd2-u0g9g1i5r8s0n6y/router-854fb99ff8-k2fd5 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-16T15:24:56Z\n Ended: 2026-09-16T15:50:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7769984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss65hn0kjsbu0s5o2aeg7qk5e1ajkjb-v20260901\", pod=\"router-78df7bd99d-57jjt\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss65hn0kjsbu0s5o2aeg7qk5e1ajkjb-v20260901/router-78df7bd99d-57jjt has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-16T15:24:56Z\n Ended: 2026-09-16T15:30:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7769984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss6623gm9lp6mh9kjc1ql22lej4dllq-oidc-wi-cluster\", pod=\"router-d46bb467b-5g27g\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss6623gm9lp6mh9kjc1ql22lej4dllq-oidc-wi-cluster/router-d46bb467b-5g27g has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-16T15:24:56Z\n Ended: 2026-09-16T15:44:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7769984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss65f27i0n6ratrctdqlr1g1ugtvfe8-v8e3q9j6q4c5y2d\", pod=\"router-6c9c647b4-m7ffl\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss65f27i0n6ratrctdqlr1g1ugtvfe8-v8e3q9j6q4c5y2d/router-6c9c647b4-m7ffl has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-16T15:26:56Z\n Ended: 2026-09-16T15:48:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7769984-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss65f7fgkikv4rdlo37t3t3p2g7ndo6-e7h0d1m9e7x8u6b\", pod=\"router-776ccdc86d-stwfg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss65f7fgkikv4rdlo37t3t3p2g7ndo6-e7h0d1m9e7x8u6b/router-776ccdc86d-stwfg has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-16T15:26:56Z\n Ended: 2026-09-16T15:54:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7769984-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss65fb8rtujqr3jelbpv31ep2qkd2cj-m1n3i0x2v3m7r0r\", pod=\"router-58d4f4f4c-spsms\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss65fb8rtujqr3jelbpv31ep2qkd2cj-m1n3i0x2v3m7r0r/router-58d4f4f4c-spsms has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"d40033bc2fc575b8a52e1ae3f692f8d892b493a3842370fdf49db7f4dbc95401","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","failed_at":"e2e","signature_id":"f53d0a32dc007711392e3ac9a6346e4050a79f13a86cf1b09a36647de1c455fb","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:584]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101ca40\u003e: \n failed to create HCP cluster np-downgrade-ptnt6v: failed waiting for cluster=\"np-downgrade-ptnt6v\" in resourcegroup=\"rg-np-downgrade-ptnt6v-s6586bctkvvs\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b81923cc-2583-43fb-a0ea-baa197fb0d6f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b81923cc-2583-43fb-a0ea-baa197fb0d6f\",\n \"name\": \"b81923cc-2583-43fb-a0ea-baa197fb0d6f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:07:45.747660708Z\",\n \"endTime\": \"2026-09-16T15:36:50.286764481Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:584]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101ca40\u003e: \n failed to create HCP cluster np-downgrade-ptnt6v: failed waiting for cluster=\"np-downgrade-ptnt6v\" in resourcegroup=\"rg-np-downgrade-ptnt6v-s6586bctkvvs\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b81923cc-2583-43fb-a0ea-baa197fb0d6f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b81923cc-2583-43fb-a0ea-baa197fb0d6f\",\n \"name\": \"b81923cc-2583-43fb-a0ea-baa197fb0d6f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:07:45.747660708Z\",\n \"endTime\": \"2026-09-16T15:36:50.286764481Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e7cade6cdb20bf44a9c9646038baa6b37e09f7e35bc67976c0f841aea8a03e76","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","failed_at":"e2e","signature_id":"ea6f45f7f8317d27652ca2ba1a9ea637325a720fb6e451e052632539521bf95b","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ca4420\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-nspddm98jhdn\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9a021ad-4d63-43da-b693-4efe761edec6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9a021ad-4d63-43da-b693-4efe761edec6\",\n \"name\": \"c9a021ad-4d63-43da-b693-4efe761edec6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:07:40.588728037Z\",\n \"endTime\": \"2026-09-16T15:36:40.298013958Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ca4420\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-nspddm98jhdn\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9a021ad-4d63-43da-b693-4efe761edec6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9a021ad-4d63-43da-b693-4efe761edec6\",\n \"name\": \"c9a021ad-4d63-43da-b693-4efe761edec6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:07:40.588728037Z\",\n \"endTime\": \"2026-09-16T15:36:40.298013958Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ef229474f840a523817784d97038339a5d090b9eb92ac5329e7778bb90f64c1e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","failed_at":"e2e","signature_id":"52b66be1d91cf6aa4302f446e0cb6737cf15f6d2f47af03effef5a56fd561fbd","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-f27q54rmwchw/cluster-candidate-4-21-s5z6tn should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f28500\u003e: \n failed to create HCP cluster cluster-candidate-4-21-s5z6tn: failed waiting for cluster=\"cluster-candidate-4-21-s5z6tn\" in resourcegroup=\"rg-candidate-4-21-f27q54rmwchw\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/336966d4-bc9c-4965-a943-7c74b3b76a9a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/336966d4-bc9c-4965-a943-7c74b3b76a9a\",\n \"name\": \"336966d4-bc9c-4965-a943-7c74b3b76a9a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:07:40.621415385Z\",\n \"endTime\": \"2026-09-16T15:36:40.360524943Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 16m59s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-f27q54rmwchw/cluster-candidate-4-21-s5z6tn should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f28500\u003e: \n failed to create HCP cluster cluster-candidate-4-21-s5z6tn: failed waiting for cluster=\"cluster-candidate-4-21-s5z6tn\" in resourcegroup=\"rg-candidate-4-21-f27q54rmwchw\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/336966d4-bc9c-4965-a943-7c74b3b76a9a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/336966d4-bc9c-4965-a943-7c74b3b76a9a\",\n \"name\": \"336966d4-bc9c-4965-a943-7c74b3b76a9a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:07:40.621415385Z\",\n \"endTime\": \"2026-09-16T15:36:40.360524943Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 16m59s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f567fe78428403c74c3b1ae01d4a23ff50e7933e3c567566c1e6802c09179ed7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","failed_at":"e2e","signature_id":"f366c8a9d3b487352e0d087e494e88beecbef611270fb1a6b15974fec8f76394","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0009db7a0\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-wngrxcmwz6zh\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20260901AndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-wngrxcmwz6zh, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0009db7a0\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-wngrxcmwz6zh\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20260901AndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-wngrxcmwz6zh, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-wngrxcmwz6zh\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20260901AndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-wngrxcmwz6zh, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":27,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","fe6b01560ac49646fd0a46c48f672cf2fc0ebbf844ea4576bb0cab7a46df7ce8"],"matched_occurrences":27,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100228886901035008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6977,"pr_state":"closed","pr_sha":"557c04379e033b9de0bd85904b49103f2cdef1b1","final_merged_sha":"557c04379e033b9de0bd85904b49103f2cdef1b1","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T14:22:39Z","started_at":"2026-09-16T14:22:39Z","completed_at":"2026-09-16T15:21:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:05.811608568Z","timing_metadata_checked_at":"2026-09-18T15:41:05.811608568Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:05.818062191Z","region_metadata_checked_at":"2026-09-18T15:41:05.818062191Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"632f09d6cbc64b89fdcb2b1db12c54f90eb11e8dd916d4e1077d2b0b7703d7d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100228886901035008","occurred_at":"2026-09-16T14:22:39Z","failed_at":"provision","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","test_suite":"step graph","failure_text":"time=2026-09-16T14:44:36.842Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-16T14:44:37.777Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-09-16T14:44:38.583Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=00e0f6eaf1a87ba23e25957419627b2f2bf902c90c40b4a9a0680ec1b94026a9 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1035008-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F00e0f6eaf1a87ba23e25957419627b2f2bf902c90c40b4a9a0680ec1b94026a9\ntime=2026-09-16T15:14:36.624Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100228663373991936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6856,"pr_state":"closed","pr_sha":"2830b6a30ee331e63a18fe2c5126d85c09e20edf","final_merged_sha":"96f38fd5a277adea0aeec323ef5685124af19024","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T14:21:48Z","started_at":"2026-09-16T14:21:48Z","completed_at":"2026-09-16T16:16:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:26.539192174Z","timing_metadata_checked_at":"2026-09-18T15:40:26.539192174Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:26.54354859Z","region_metadata_checked_at":"2026-09-18T15:40:26.54354859Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"a3dc6af153fd81555fbd7f0e0e1714640e6171bd6957ee5562dbfc14aeeb9a3c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100228663373991936","occurred_at":"2026-09-16T14:21:48Z","failed_at":"e2e","signature_id":"27221439d7603c27de714a5d301e36e3ab9f94a39958b4fd4679cc00d417d51d","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eec980\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-s5qsjcnlw5bb\" to finish creating: GET https://rp.j3991936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/91f7755d-f9d4-4444-beb5-36084a84bad6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/91f7755d-f9d4-4444-beb5-36084a84bad6\",\n \"name\": \"91f7755d-f9d4-4444-beb5-36084a84bad6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:07:08.748392201Z\",\n \"endTime\": \"2026-09-16T15:36:08.491792719Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m27s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eec980\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-s5qsjcnlw5bb\" to finish creating: GET https://rp.j3991936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/91f7755d-f9d4-4444-beb5-36084a84bad6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/91f7755d-f9d4-4444-beb5-36084a84bad6\",\n \"name\": \"91f7755d-f9d4-4444-beb5-36084a84bad6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:07:08.748392201Z\",\n \"endTime\": \"2026-09-16T15:36:08.491792719Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m27s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b2e23b1dcc8961697efc8d99cd78be01fe605c4226ab2959f64fed6823ec8c26","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100228663373991936","occurred_at":"2026-09-16T14:21:48Z","failed_at":"alert","signature_id":"af0482283bab09054590813e51480bea798a5743b7bcc33fa9c63b58d8645bbe","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T15:09:29Z\n Ended: 2026-09-16T15:39:29Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3991936-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3991936-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100227428122103808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6439,"pr_state":"open","pr_sha":"b6b00cdc8ea3ee249a9faa811955e4eb095652ef","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T14:16:53Z","started_at":"2026-09-16T14:16:53Z","completed_at":"2026-09-16T16:18:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:06.395010287Z","timing_metadata_checked_at":"2026-09-18T15:40:06.395010287Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:06.398302199Z","region_metadata_checked_at":"2026-09-18T15:40:06.398302199Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f55c19e6bc2f932e7d708c3d486464b5936ca6ccdc6f3a8fa73ec314adfbe5f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100227428122103808","occurred_at":"2026-09-16T14:16:53Z","failed_at":"e2e","signature_id":"0240604425a3af254fe1ce6e815c04f07285897c6355ead3254c138853f560f1","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e74c20\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-xzp2sp8wgrn2\" to finish creating: GET https://rp.j2103808.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c63d988d-abe6-4ea1-9d3f-acc959502c90\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c63d988d-abe6-4ea1-9d3f-acc959502c90\",\n \"name\": \"c63d988d-abe6-4ea1-9d3f-acc959502c90\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:04:26.332691883Z\",\n \"endTime\": \"2026-09-16T15:33:30.087028134Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 15m41s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e74c20\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-xzp2sp8wgrn2\" to finish creating: GET https://rp.j2103808.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c63d988d-abe6-4ea1-9d3f-acc959502c90\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c63d988d-abe6-4ea1-9d3f-acc959502c90\",\n \"name\": \"c63d988d-abe6-4ea1-9d3f-acc959502c90\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:04:26.332691883Z\",\n \"endTime\": \"2026-09-16T15:33:30.087028134Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 15m41s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6983,"pr_state":"open","pr_sha":"030ffa2ce8af642ed82f8857f8f2da3cdb834fa4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T14:14:56Z","started_at":"2026-09-16T14:14:56Z","completed_at":"2026-09-16T16:18:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:08.926877389Z","timing_metadata_checked_at":"2026-09-18T15:41:08.926877389Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:08.930329301Z","region_metadata_checked_at":"2026-09-18T15:41:08.930329301Z"},"failed_at":["alert","e2e"],"failed_test_count":18,"occurrences":[{"row_id":"028cf6e1bb203a0ca397a2107374f6d2c2548985e2c69c3d1ec2a7fca2fbeb32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","failed_at":"e2e","signature_id":"32832ce99401d97a958b7dcdfe20140118a267411f3e842c201974409fd354fb","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001712108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-z7jd9d5bkv4x\" to finish deleting: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fe3c0be1-f3f8-476b-b13f-1ceef9fd8976\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fe3c0be1-f3f8-476b-b13f-1ceef9fd8976\",\n \"name\": \"fe3c0be1-f3f8-476b-b13f-1ceef9fd8976\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:23:43.887981496Z\",\n \"endTime\": \"2026-09-16T15:47:44.543137714Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss654fr7qth67si53recd86bd0moi28 still exists (deletion dispatched at 2026-09-16T15:23:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001712108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-z7jd9d5bkv4x\" to finish deleting: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fe3c0be1-f3f8-476b-b13f-1ceef9fd8976\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fe3c0be1-f3f8-476b-b13f-1ceef9fd8976\",\n \"name\": \"fe3c0be1-f3f8-476b-b13f-1ceef9fd8976\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:23:43.887981496Z\",\n \"endTime\": \"2026-09-16T15:47:44.543137714Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss654fr7qth67si53recd86bd0moi28 still exists (deletion dispatched at 2026-09-16T15:23:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0fb96287d90670d2148a8e4141199e5185f8162fc41896509e7b36fe7530f1d8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","failed_at":"e2e","signature_id":"416d3122655e04bc6a1e6cd87fa70c5433f7bca4d0212f66aecf79af52b1d0a5","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fcac20\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-jf5j95vczmzx\" to finish creating: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c06c517-46e6-4d48-93f9-026c39ae1338\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c06c517-46e6-4d48-93f9-026c39ae1338\",\n \"name\": \"9c06c517-46e6-4d48-93f9-026c39ae1338\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:05:48.234554059Z\",\n \"endTime\": \"2026-09-16T15:34:54.47386127Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m55s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fcac20\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-jf5j95vczmzx\" to finish creating: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c06c517-46e6-4d48-93f9-026c39ae1338\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c06c517-46e6-4d48-93f9-026c39ae1338\",\n \"name\": \"9c06c517-46e6-4d48-93f9-026c39ae1338\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:05:48.234554059Z\",\n \"endTime\": \"2026-09-16T15:34:54.47386127Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m55s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1795fa4e81c7b4ab9edc7db85f82634f0a1ef79576c9e5fa7baa387124a689ad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","failed_at":"e2e","signature_id":"b9b50758a1aa006036c7c1ee58af90cd509c7b1900722ef0229b46ed2f5e8563","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"18ead0d5832e482a62ea5f23505e63e202fbeb4faa7480dd475e861a48a1c5f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","failed_at":"e2e","signature_id":"773504a23a339bedaebd473b20c28b68aef28fa8120eb503e3c2ada5990e4cf7","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e3a9a0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-9r6rpqjgz268\" to finish creating: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb680db4-320e-4707-9f4b-0f2cd699ec63\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb680db4-320e-4707-9f4b-0f2cd699ec63\",\n \"name\": \"cb680db4-320e-4707-9f4b-0f2cd699ec63\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:05:59.161824665Z\",\n \"endTime\": \"2026-09-16T15:35:04.514571365Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 24m8s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e3a9a0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-9r6rpqjgz268\" to finish creating: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb680db4-320e-4707-9f4b-0f2cd699ec63\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb680db4-320e-4707-9f4b-0f2cd699ec63\",\n \"name\": \"cb680db4-320e-4707-9f4b-0f2cd699ec63\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:05:59.161824665Z\",\n \"endTime\": \"2026-09-16T15:35:04.514571365Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 24m8s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1a2dc0a7f3f8cdef5800b0790a68d59a57e0110e7104b535b2f757a80b4d83a5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","failed_at":"e2e","signature_id":"42ea74c98dd160984e4c829f799e27d45bf2b5018f73c2238f76e9d8d6a3f703","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001094a00\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-nnpsp9gzr6sk\" to finish creating: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d3a50c81-c043-4ac7-a378-fa722f63c462\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d3a50c81-c043-4ac7-a378-fa722f63c462\",\n \"name\": \"d3a50c81-c043-4ac7-a378-fa722f63c462\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:05:49.08539059Z\",\n \"endTime\": \"2026-09-16T15:34:54.502329591Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m17s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001094a00\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-nnpsp9gzr6sk\" to finish creating: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d3a50c81-c043-4ac7-a378-fa722f63c462\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d3a50c81-c043-4ac7-a378-fa722f63c462\",\n \"name\": \"d3a50c81-c043-4ac7-a378-fa722f63c462\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:05:49.08539059Z\",\n \"endTime\": \"2026-09-16T15:34:54.502329591Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m17s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"261f84a340177836372258f80807b6215a0ea0862a4373a1e9e277c6cb7acd6e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","failed_at":"alert","signature_id":"9efd97c6c35dce815008610c2dc8e01945d6be74819a30c6714b0f4b7f9ad9da","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T15:28:20Z\n Ended: 2026-09-16T15:40:21Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j8680064-svc\", component=\"backend\", name=\"dataplaneoidcfederation\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue dataplaneoidcfederation has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired"}},{"row_id":"2a030ae322790e2c0892e8996e28cb5b6abc42a46d7eb08201efa400182bb897","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","failed_at":"e2e","signature_id":"6422616296d3a5a4112b796133cccebf1e701aef5be2522f1440d40d7f73f94f","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001532060\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-g8h2hwb78d6v\" to finish deleting: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/acb2e182-ba85-4e97-b275-c6baf85458ad\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/acb2e182-ba85-4e97-b275-c6baf85458ad\",\n \"name\": \"acb2e182-ba85-4e97-b275-c6baf85458ad\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:22:20.960856273Z\",\n \"endTime\": \"2026-09-16T15:46:24.513434488Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss64nns1ma3p3sappn3d83v1kq4u4mc still exists (deletion dispatched at 2026-09-16T15:22:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001532060\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-g8h2hwb78d6v\" to finish deleting: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/acb2e182-ba85-4e97-b275-c6baf85458ad\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/acb2e182-ba85-4e97-b275-c6baf85458ad\",\n \"name\": \"acb2e182-ba85-4e97-b275-c6baf85458ad\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:22:20.960856273Z\",\n \"endTime\": \"2026-09-16T15:46:24.513434488Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss64nns1ma3p3sappn3d83v1kq4u4mc still exists (deletion dispatched at 2026-09-16T15:22:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"334c422d1f3b0459175b899849152b6116e62e67916f4226ed8f3f27545c2e51","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","failed_at":"alert","signature_id":"fe76c2aef36843354f881968133b4c2405287f6a2f2ee3e1e7ab8ab4682897ce","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T15:24:31Z\n Ended: 2026-09-16T15:42:32Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8680064-svc\", component=\"backend\", name=\"dataplaneoidcfederation\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue dataplaneoidcfederation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"3f405ef8b5fe788b890b8ae9227386ddd97f8a2fe7d5a02214cb3a3deb3bc272","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","failed_at":"e2e","signature_id":"37839b6f366af376a2e2dee46f1b956ca9ee9897a768408f797bc256ab2e722b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-96ddmv\" for cluster \"cluster-candidate-5-0-96ddmv\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d846a0\u003e: \n failed to create NodePool np-96ddmv: failed waiting for nodepool=\"np-96ddmv\" for cluster \"cluster-candidate-5-0-96ddmv\" in resourcegroup=\"rg-candidate-5-0-dxgjw5725ck7\" to finish creating: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75dfa262-dc67-4818-8d1e-eacd838b47fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75dfa262-dc67-4818-8d1e-eacd838b47fd\",\n \"name\": \"75dfa262-dc67-4818-8d1e-eacd838b47fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:23:26.413052673Z\",\n \"endTime\": \"2026-09-16T15:24:04.373688035Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-96ddmv\" for cluster \"cluster-candidate-5-0-96ddmv\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d846a0\u003e: \n failed to create NodePool np-96ddmv: failed waiting for nodepool=\"np-96ddmv\" for cluster \"cluster-candidate-5-0-96ddmv\" in resourcegroup=\"rg-candidate-5-0-dxgjw5725ck7\" to finish creating: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75dfa262-dc67-4818-8d1e-eacd838b47fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75dfa262-dc67-4818-8d1e-eacd838b47fd\",\n \"name\": \"75dfa262-dc67-4818-8d1e-eacd838b47fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:23:26.413052673Z\",\n \"endTime\": \"2026-09-16T15:24:04.373688035Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"61673dbbdca7d11f9bb6ce9024fe1cd316bf02dd7f38e1173b628a2ee5c8c954","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","failed_at":"e2e","signature_id":"ea4b0a4eb2c285656f848886af6bb6a27818c286a7b34ef24973fc4e223cf7a2","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001880090\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-pp5nvw5grtc7\" to finish deleting: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/286d07fb-7bc7-4ce1-aafb-d75cdb5dbbfc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/286d07fb-7bc7-4ce1-aafb-d75cdb5dbbfc\",\n \"name\": \"286d07fb-7bc7-4ce1-aafb-d75cdb5dbbfc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:23:13.971647999Z\",\n \"endTime\": \"2026-09-16T15:47:14.615898434Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss65qamt1ukuh6foc29hal8kisg465u still exists (deletion dispatched at 2026-09-16T15:23:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001880090\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-pp5nvw5grtc7\" to finish deleting: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/286d07fb-7bc7-4ce1-aafb-d75cdb5dbbfc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/286d07fb-7bc7-4ce1-aafb-d75cdb5dbbfc\",\n \"name\": \"286d07fb-7bc7-4ce1-aafb-d75cdb5dbbfc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:23:13.971647999Z\",\n \"endTime\": \"2026-09-16T15:47:14.615898434Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss65qamt1ukuh6foc29hal8kisg465u still exists (deletion dispatched at 2026-09-16T15:23:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"636ec93d38604b55a6a046a915a4f10560175ca1cc6deb014fbdc93546ba2b17","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","failed_at":"e2e","signature_id":"e114928abafbdcce9fc1b5a0f9590defe570e1bf5ceed7138ffdb8bb94282914","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000a101b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-d7xf2jghkngg\" to finish deleting: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fe4bbd3c-2ce0-4443-b660-f1afc9b9ff48\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fe4bbd3c-2ce0-4443-b660-f1afc9b9ff48\",\n \"name\": \"fe4bbd3c-2ce0-4443-b660-f1afc9b9ff48\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:25:39.570793353Z\",\n \"endTime\": \"2026-09-16T15:49:44.508102649Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss64gl4enpi00gf4b4hmbe2u4tjukt5 still exists (deletion dispatched at 2026-09-16T15:25:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000a101b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-d7xf2jghkngg\" to finish deleting: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fe4bbd3c-2ce0-4443-b660-f1afc9b9ff48\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fe4bbd3c-2ce0-4443-b660-f1afc9b9ff48\",\n \"name\": \"fe4bbd3c-2ce0-4443-b660-f1afc9b9ff48\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:25:39.570793353Z\",\n \"endTime\": \"2026-09-16T15:49:44.508102649Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss64gl4enpi00gf4b4hmbe2u4tjukt5 still exists (deletion dispatched at 2026-09-16T15:25:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7521008dbd001c8e3428fe950ebf562d2c464bf2df9cc7ed8fede2e4f2e52113","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","failed_at":"e2e","signature_id":"e94f3f5e69e8e136647e4790e270a1391748aa6703a074a6fa38b2687beb9db4","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001660078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-swrr5k\" in resourcegroup=\"rg-zstream-upgrade-4-21-xx2vdc7swlps\" to finish deleting: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/81df39bb-b3b6-42b1-a83c-57dc3461bd95\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/81df39bb-b3b6-42b1-a83c-57dc3461bd95\",\n \"name\": \"81df39bb-b3b6-42b1-a83c-57dc3461bd95\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:30:38.197952822Z\",\n \"endTime\": \"2026-09-16T15:54:44.460453861Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss64iqhe77hj84b81d1vg9nt6d0dhsr still exists (deletion dispatched at 2026-09-16T15:30:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001660078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-swrr5k\" in resourcegroup=\"rg-zstream-upgrade-4-21-xx2vdc7swlps\" to finish deleting: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/81df39bb-b3b6-42b1-a83c-57dc3461bd95\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/81df39bb-b3b6-42b1-a83c-57dc3461bd95\",\n \"name\": \"81df39bb-b3b6-42b1-a83c-57dc3461bd95\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:30:38.197952822Z\",\n \"endTime\": \"2026-09-16T15:54:44.460453861Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss64iqhe77hj84b81d1vg9nt6d0dhsr still exists (deletion dispatched at 2026-09-16T15:30:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8b838a7fa4640674addff050d5a33289cae6fbde4d40802152b2cbd82a11e2e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","failed_at":"alert","signature_id":"4778f344e9b25530d8b9156641a99f6b2838c230f7532738e3f69e081b9f7a7e","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T15:16:31Z\n Ended: 2026-09-16T16:05:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8680064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss64a2ngrrh8v43g4224kkvsoq6vf20-l5z4r5n6l2r7j5q\", pod=\"router-686dcf6999-wvnqc\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss64a2ngrrh8v43g4224kkvsoq6vf20-l5z4r5n6l2r7j5q/router-686dcf6999-wvnqc has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-16T15:17:31Z\n Ended: 2026-09-16T15:49:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8680064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss64lsro1oea3d8uqc5rdlv4i8bfn9s-n0h3v6f4v6w5e2y\", pod=\"router-656766fdcc-4gn7f\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss64lsro1oea3d8uqc5rdlv4i8bfn9s-n0h3v6f4v6w5e2y/router-656766fdcc-4gn7f has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-16T15:18:31Z\n Ended: 2026-09-16T15:45:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8680064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss64j8k24ndk4j4jets482r57jmq464-private-ingress\", pod=\"router-8698f9c9d5-f9464\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss64j8k24ndk4j4jets482r57jmq464-private-ingress/router-8698f9c9d5-f9464 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-16T15:18:31Z\n Ended: 2026-09-16T15:44:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8680064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss64jdehrrlcdafsmr824r6gu6d4v3i-k5y9i2m2i7c3a6z\", pod=\"router-684b9dbc6c-mknm9\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss64jdehrrlcdafsmr824r6gu6d4v3i-k5y9i2m2i7c3a6z/router-684b9dbc6c-mknm9 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"90c6a2c84b28094f8eedf4b269f6943806c66b7209064b75faa535f50438598f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","failed_at":"e2e","signature_id":"b79349f0a2b72f52e04da9ac4e8ef7179dd24862a80796349270662d77641269","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000012e40\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-wbh9jqvv44p7\" to finish deleting: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9db0308e-c5ac-4693-b367-d76dc77af863\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9db0308e-c5ac-4693-b367-d76dc77af863\",\n \"name\": \"9db0308e-c5ac-4693-b367-d76dc77af863\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:29:44.443897591Z\",\n \"endTime\": \"2026-09-16T15:53:44.471117038Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss64i7i766p1sol3r0dalpub5p7l53h still exists (deletion dispatched at 2026-09-16T15:29:46Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000012e40\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-wbh9jqvv44p7\" to finish deleting: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9db0308e-c5ac-4693-b367-d76dc77af863\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9db0308e-c5ac-4693-b367-d76dc77af863\",\n \"name\": \"9db0308e-c5ac-4693-b367-d76dc77af863\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:29:44.443897591Z\",\n \"endTime\": \"2026-09-16T15:53:44.471117038Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss64i7i766p1sol3r0dalpub5p7l53h still exists (deletion dispatched at 2026-09-16T15:29:46Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"63a24fc72ea6be67478ba57ceca4f83bfef60aaeae2826b2abf0c75f70bec403","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9b234fe59b0c5cd732fc076f85444d0419133a8cf88b50af97162dbc4075d3c1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","failed_at":"e2e","signature_id":"97a8d855655aa481b47156e39fc63195c1bf1486f4d7c41cbeb84f0001f75fe9","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001006f60\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-gpg49rkjvb2m\" to finish creating: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c60438e7-1f3d-4f9d-a2a0-7dd97f6e3d9b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c60438e7-1f3d-4f9d-a2a0-7dd97f6e3d9b\",\n \"name\": \"c60438e7-1f3d-4f9d-a2a0-7dd97f6e3d9b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:05:45.793133162Z\",\n \"endTime\": \"2026-09-16T15:34:54.478500072Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 20m28s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001006f60\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-gpg49rkjvb2m\" to finish creating: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c60438e7-1f3d-4f9d-a2a0-7dd97f6e3d9b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c60438e7-1f3d-4f9d-a2a0-7dd97f6e3d9b\",\n \"name\": \"c60438e7-1f3d-4f9d-a2a0-7dd97f6e3d9b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:05:45.793133162Z\",\n \"endTime\": \"2026-09-16T15:34:54.478500072Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 20m28s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a9766dcb6c2be12273c97722a8f7d2fc8796935aa177bac0b9e0ddecedb70122","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","failed_at":"e2e","signature_id":"d1ada8baec3cfc5a8db7ecb4d82edd81b7f41fd13101da84520ce2bb5cd103d1","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004e6380\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-z42dcrsxlj77\" to finish creating: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6cbd1f6e-1e54-401b-8dd5-b1aea43e3a87\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6cbd1f6e-1e54-401b-8dd5-b1aea43e3a87\",\n \"name\": \"6cbd1f6e-1e54-401b-8dd5-b1aea43e3a87\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:07:22.469453633Z\",\n \"endTime\": \"2026-09-16T15:36:24.366773591Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 20m57s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004e6380\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-z42dcrsxlj77\" to finish creating: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6cbd1f6e-1e54-401b-8dd5-b1aea43e3a87\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6cbd1f6e-1e54-401b-8dd5-b1aea43e3a87\",\n \"name\": \"6cbd1f6e-1e54-401b-8dd5-b1aea43e3a87\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:07:22.469453633Z\",\n \"endTime\": \"2026-09-16T15:36:24.366773591Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 20m57s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ad53033c5d8b6fdc00005e6aeee80fc839d1bcc77b1e3760e3b5674c5d40a8be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","failed_at":"e2e","signature_id":"47db8067bcef5f852d6be174031d8a4891925f676b5063754561044e04b33b85","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011a4060\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-k6j48pxdpt2p\" to finish deleting: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a6e3e29a-385a-455a-b41d-893521143636\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a6e3e29a-385a-455a-b41d-893521143636\",\n \"name\": \"a6e3e29a-385a-455a-b41d-893521143636\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:28:48.608443563Z\",\n \"endTime\": \"2026-09-16T15:52:54.427688455Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss657krpigsh0ful51ank8n8kbiubtm still exists (deletion dispatched at 2026-09-16T15:28:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011a4060\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-k6j48pxdpt2p\" to finish deleting: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a6e3e29a-385a-455a-b41d-893521143636\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a6e3e29a-385a-455a-b41d-893521143636\",\n \"name\": \"a6e3e29a-385a-455a-b41d-893521143636\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:28:48.608443563Z\",\n \"endTime\": \"2026-09-16T15:52:54.427688455Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss657krpigsh0ful51ank8n8kbiubtm still exists (deletion dispatched at 2026-09-16T15:28:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fd5c6c59dcd09c52726f9b93690a2d2b67195371b9436a073be632622a8aeacd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","failed_at":"e2e","signature_id":"4634c0871869430762696feaab7758bce3c5d6614cf0816c70b2c3b8b4d3ac0d","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010207a0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-z5wdv6h4vdnz\" to finish creating: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ba551b96-dd27-4bf9-b784-21cb42b2d916\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ba551b96-dd27-4bf9-b784-21cb42b2d916\",\n \"name\": \"ba551b96-dd27-4bf9-b784-21cb42b2d916\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:08:31.358252502Z\",\n \"endTime\": \"2026-09-16T15:37:34.476730603Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m7s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010207a0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-z5wdv6h4vdnz\" to finish creating: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ba551b96-dd27-4bf9-b784-21cb42b2d916\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ba551b96-dd27-4bf9-b784-21cb42b2d916\",\n \"name\": \"ba551b96-dd27-4bf9-b784-21cb42b2d916\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:08:31.358252502Z\",\n \"endTime\": \"2026-09-16T15:37:34.476730603Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m7s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":18,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","63a24fc72ea6be67478ba57ceca4f83bfef60aaeae2826b2abf0c75f70bec403","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":18,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222905899028480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6940,"pr_state":"closed","pr_sha":"62cdd448d25099f3b77adb5f5e21439e857c6d5f","final_merged_sha":"62cdd448d25099f3b77adb5f5e21439e857c6d5f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-16T13:58:55Z","started_at":"2026-09-16T13:58:55Z","completed_at":"2026-09-16T16:23:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:45.751139694Z","timing_metadata_checked_at":"2026-09-18T15:40:45.751139694Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:45.754152305Z","region_metadata_checked_at":"2026-09-18T15:40:45.754152305Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222891458039808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6952,"pr_state":"closed","pr_sha":"a4de05dadb773673a7d1a63ee2677dbb182bda10","final_merged_sha":"a4de05dadb773673a7d1a63ee2677dbb182bda10","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T13:58:52Z","started_at":"2026-09-16T13:58:52Z","completed_at":"2026-09-16T16:10:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:57.252792562Z","timing_metadata_checked_at":"2026-09-18T15:40:57.252792562Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:57.256219174Z","region_metadata_checked_at":"2026-09-18T15:40:57.256219174Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"0a590169daca9371ad5b6825cf3120cc44037a7e7903d484e1bc18b42da64170","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222891458039808","occurred_at":"2026-09-16T13:58:52Z","failed_at":"e2e","signature_id":"8491d54e1e6f9d99de4d3a50c4e2d20f3eb57611f24ff8bb3d09a72d4b3c8424","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0017c8048\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-jxshqr\" in resourcegroup=\"rg-zstream-upgrade-4-20-h67wq8dhmgts\" to finish deleting: GET https://rp.j8039808.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b1e26b2-b451-4f5e-9c4e-f445f49d6b86\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b1e26b2-b451-4f5e-9c4e-f445f49d6b86\",\n \"name\": \"5b1e26b2-b451-4f5e-9c4e-f445f49d6b86\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:13:10.410853288Z\",\n \"endTime\": \"2026-09-16T15:37:12.377929551Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss5vh867atil535pho6oarttl9i35ou still exists (deletion dispatched at 2026-09-16T15:13:20Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0017c8048\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-jxshqr\" in resourcegroup=\"rg-zstream-upgrade-4-20-h67wq8dhmgts\" to finish deleting: GET https://rp.j8039808.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b1e26b2-b451-4f5e-9c4e-f445f49d6b86\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b1e26b2-b451-4f5e-9c4e-f445f49d6b86\",\n \"name\": \"5b1e26b2-b451-4f5e-9c4e-f445f49d6b86\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:13:10.410853288Z\",\n \"endTime\": \"2026-09-16T15:37:12.377929551Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss5vh867atil535pho6oarttl9i35ou still exists (deletion dispatched at 2026-09-16T15:13:20Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"63a24fc72ea6be67478ba57ceca4f83bfef60aaeae2826b2abf0c75f70bec403","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"732b3b506f50a2d96dbb0b37b41ffc07ee813fb0b64e16efcdd8d5c77c4810d6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222891458039808","occurred_at":"2026-09-16T13:58:52Z","failed_at":"e2e","signature_id":"4c1a786ccdf5f4a6714330ab65d6eaf1faaac9d21fed54613d33cde0056d6e52","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:144]: node pool \"np-1\" should have exactly one active version after initial creation\nExpected\n \u003c[]*armredhatopenshifthcp.NodePoolActiveVersion | len:0, cap:0\u003e: ...\nto have length 1\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:144]: node pool \"np-1\" should have exactly one active version after initial creation\nExpected\n \u003c[]*armredhatopenshifthcp.NodePoolActiveVersion | len:0, cap:0\u003e: ...\nto have length 1","failure_pattern_match":{"status":"clustered","failure_pattern_id":"662e49bfb814d1960ee5fb9e525e6ffab8495cd359fc51176893943560eee29a","failure_pattern":"node pool \"\u003cnodepool\u003e\" should have exactly one active version after initial creation","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:144]: node pool \"np-1\" should have exactly one active version after initial creation"}},{"row_id":"77e691290727ba9465f4defef3603476429ab508c70edbd4674d2adc2944a5c9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222891458039808","occurred_at":"2026-09-16T13:58:52Z","failed_at":"e2e","signature_id":"1cb320d43390d047edfe367eca720f9ae752f792b2067b9a7d668b21e611b437","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0019b8288\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-r7ljx995hlwf\" to finish deleting: GET https://rp.j8039808.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/45f270e6-1026-464f-92a5-1c2c1a33f197\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/45f270e6-1026-464f-92a5-1c2c1a33f197\",\n \"name\": \"45f270e6-1026-464f-92a5-1c2c1a33f197\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:12:09.364011371Z\",\n \"endTime\": \"2026-09-16T15:36:12.497199905Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss5vol9udvmi9p399pnj4tdfvicpu9k still exists (deletion dispatched at 2026-09-16T15:12:10Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0019b8288\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-r7ljx995hlwf\" to finish deleting: GET https://rp.j8039808.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/45f270e6-1026-464f-92a5-1c2c1a33f197\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/45f270e6-1026-464f-92a5-1c2c1a33f197\",\n \"name\": \"45f270e6-1026-464f-92a5-1c2c1a33f197\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:12:09.364011371Z\",\n \"endTime\": \"2026-09-16T15:36:12.497199905Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss5vol9udvmi9p399pnj4tdfvicpu9k still exists (deletion dispatched at 2026-09-16T15:12:10Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["63a24fc72ea6be67478ba57ceca4f83bfef60aaeae2826b2abf0c75f70bec403","662e49bfb814d1960ee5fb9e525e6ffab8495cd359fc51176893943560eee29a","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222840434331648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6976,"pr_state":"open","pr_sha":"3f37770fba22a55b4336e3a85cbf275bf37c8279","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T13:58:40Z","started_at":"2026-09-16T13:58:40Z","completed_at":"2026-09-16T16:00:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:04.711138205Z","timing_metadata_checked_at":"2026-09-18T15:41:04.711138205Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:04.714137516Z","region_metadata_checked_at":"2026-09-18T15:41:04.714137516Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"011d4189d2f05123f0618e653d80a7d712e4db9ca75f01a0cb49bcba9d19e69d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222840434331648","occurred_at":"2026-09-16T13:58:40Z","failed_at":"e2e","signature_id":"16a76a4ed4fe90d47f48a9657d0216e8d1f87acf39a8dad4d8ba5c30878fe100","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00133c168\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-8q54sb\" in resourcegroup=\"rg-candidate-4-21-96n8w9b9nmcr\" to finish deleting: GET https://rp.j4331648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8145a55-f885-4824-ade4-c78ded06b41b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8145a55-f885-4824-ade4-c78ded06b41b\",\n \"name\": \"d8145a55-f885-4824-ade4-c78ded06b41b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:09:44.33584903Z\",\n \"endTime\": \"2026-09-16T15:33:51.215190325Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss5qm47uq9tl4u0n0hlgrdb7qttgv6e still exists (deletion dispatched at 2026-09-16T15:09:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00133c168\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-8q54sb\" in resourcegroup=\"rg-candidate-4-21-96n8w9b9nmcr\" to finish deleting: GET https://rp.j4331648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8145a55-f885-4824-ade4-c78ded06b41b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8145a55-f885-4824-ade4-c78ded06b41b\",\n \"name\": \"d8145a55-f885-4824-ade4-c78ded06b41b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:09:44.33584903Z\",\n \"endTime\": \"2026-09-16T15:33:51.215190325Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss5qm47uq9tl4u0n0hlgrdb7qttgv6e still exists (deletion dispatched at 2026-09-16T15:09:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"24164a32aa2191ac9f439978ca50b54b36d4e3bb1f1e3b7f266cda7b50fd2e3e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222840434331648","occurred_at":"2026-09-16T13:58:40Z","failed_at":"alert","signature_id":"045a81b731fe6b760898ced74dc07c1acf462fd8638ae785f456d496a795842e","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 20 time(s) (2 unknown, 18 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T15:02:01Z\n Ended: 2026-09-16T15:18:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4331648-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss5qrj6l3t7svqqgghv0p32ifilmtgo-private-kas\", pod=\"router-74979f6ddd-l297b\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss5qrj6l3t7svqqgghv0p32ifilmtgo-private-kas/router-74979f6ddd-l297b has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-16T15:02:01Z\n Ended: 2026-09-16T15:18:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4331648-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss5qnvpmpmfksiqumamsk6f4a5n8h97-i8m9u7o2i9m1s4l\", pod=\"router-d798bf795-r5qm8\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss5qnvpmpmfksiqumamsk6f4a5n8h97-i8m9u7o2i9m1s4l/router-d798bf795-r5qm8 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"340a50f00b9b003c97001bfc2b1031916cca73cfe19a3ca44c686ba47b861219","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222840434331648","occurred_at":"2026-09-16T13:58:40Z","failed_at":"e2e","signature_id":"46b2146b5bea1fbbe7547a18b5b7e0a96d274edde0ce207e2338267f45f60742","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001654180\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-vwpvr2srntsx\" to finish deleting: GET https://rp.j4331648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9690275d-f0a2-4269-97f3-ff42287a50a3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9690275d-f0a2-4269-97f3-ff42287a50a3\",\n \"name\": \"9690275d-f0a2-4269-97f3-ff42287a50a3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:08:51.225823952Z\",\n \"endTime\": \"2026-09-16T15:32:51.218856846Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss5qjn2q2n7cj829v2pum0bhr8rsof2 still exists (deletion dispatched at 2026-09-16T15:08:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001654180\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-vwpvr2srntsx\" to finish deleting: GET https://rp.j4331648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9690275d-f0a2-4269-97f3-ff42287a50a3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9690275d-f0a2-4269-97f3-ff42287a50a3\",\n \"name\": \"9690275d-f0a2-4269-97f3-ff42287a50a3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:08:51.225823952Z\",\n \"endTime\": \"2026-09-16T15:32:51.218856846Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss5qjn2q2n7cj829v2pum0bhr8rsof2 still exists (deletion dispatched at 2026-09-16T15:08:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"56f26f21980a7fa6ff0b15fa5028e8b585ea036c1411395c563e13f492036c6a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222840434331648","occurred_at":"2026-09-16T13:58:40Z","failed_at":"e2e","signature_id":"080a9e45d6f4d800d1c8aca157df99a9c02fa085354fe52205b1fcea2ffcae50","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:403]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc000dfe180\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-kp7j9psbb8sj\" to finish creating: GET https://rp.j4331648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/284aa271-80bb-4df5-aaf0-7131d6eceb09\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/284aa271-80bb-4df5-aaf0-7131d6eceb09\",\n \"name\": \"284aa271-80bb-4df5-aaf0-7131d6eceb09\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:12:21.082535677Z\",\n \"endTime\": \"2026-09-16T15:13:01.614774274Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:403]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc000dfe180\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-kp7j9psbb8sj\" to finish creating: GET https://rp.j4331648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/284aa271-80bb-4df5-aaf0-7131d6eceb09\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/284aa271-80bb-4df5-aaf0-7131d6eceb09\",\n \"name\": \"284aa271-80bb-4df5-aaf0-7131d6eceb09\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:12:21.082535677Z\",\n \"endTime\": \"2026-09-16T15:13:01.614774274Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5bb20b0379dbd7a863b4c094acbf5f7bc5de901302cb50af0eed4e35e0705a3c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222840434331648","occurred_at":"2026-09-16T13:58:40Z","failed_at":"e2e","signature_id":"3b8fdb12977219a6ee2876ab164d9d979ba1367fe5e61c35ac3016ab87e10d51","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fec640\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-9qq8dbbbpq99\" to finish creating: GET https://rp.j4331648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c197dc3a-1727-4935-ab5c-4e20220390ea\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c197dc3a-1727-4935-ab5c-4e20220390ea\",\n \"name\": \"c197dc3a-1727-4935-ab5c-4e20220390ea\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T14:44:47.197696304Z\",\n \"endTime\": \"2026-09-16T15:13:51.528371822Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 17m44s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fec640\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-9qq8dbbbpq99\" to finish creating: GET https://rp.j4331648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c197dc3a-1727-4935-ab5c-4e20220390ea\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c197dc3a-1727-4935-ab5c-4e20220390ea\",\n \"name\": \"c197dc3a-1727-4935-ab5c-4e20220390ea\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T14:44:47.197696304Z\",\n \"endTime\": \"2026-09-16T15:13:51.528371822Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 17m44s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a7ff0a46864c1a8cc8866c213f98e31dcfe4efcbe7169f25a9e92ab1a0228b0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222840434331648","occurred_at":"2026-09-16T13:58:40Z","failed_at":"e2e","signature_id":"162026bfd655fc4db32e7049c532e8d81f8bd34a8e4e6625145cd5a31c7568a3","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0011eef40\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-gn4q6q4pqf7b\" to finish creating: GET https://rp.j4331648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c302cb08-5816-4f3c-a786-daa45a7d6406\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c302cb08-5816-4f3c-a786-daa45a7d6406\",\n \"name\": \"c302cb08-5816-4f3c-a786-daa45a7d6406\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:12:22.418838081Z\",\n \"endTime\": \"2026-09-16T15:13:01.450563178Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0011eef40\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-gn4q6q4pqf7b\" to finish creating: GET https://rp.j4331648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c302cb08-5816-4f3c-a786-daa45a7d6406\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c302cb08-5816-4f3c-a786-daa45a7d6406\",\n \"name\": \"c302cb08-5816-4f3c-a786-daa45a7d6406\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T15:12:22.418838081Z\",\n \"endTime\": \"2026-09-16T15:13:01.450563178Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-gn4q6q4pqf7b\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100221930895314944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6894,"pr_state":"closed","pr_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","final_merged_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-16T13:55:02Z","started_at":"2026-09-16T13:55:02Z","completed_at":"2026-09-16T15:54:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:33.461195672Z","timing_metadata_checked_at":"2026-09-18T15:40:33.461195672Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:33.464251583Z","region_metadata_checked_at":"2026-09-18T15:40:33.464251583Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100214915997372416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6920,"pr_state":"open","pr_sha":"4a210f75791b1343c03bcca6d110eba2b769b1f5","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-16T13:27:10Z","started_at":"2026-09-16T13:27:10Z","completed_at":"2026-09-16T15:26:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:40.391631507Z","timing_metadata_checked_at":"2026-09-18T15:40:40.391631507Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:40.395063619Z","region_metadata_checked_at":"2026-09-18T15:40:40.395063619Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100213017273700352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6968,"pr_state":"open","pr_sha":"3111633542bf1771939a60beec0032fd47dcc032","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T13:19:37Z","started_at":"2026-09-16T13:19:37Z","completed_at":"2026-09-16T15:13:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:00.898710176Z","timing_metadata_checked_at":"2026-09-18T15:41:00.898710176Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:00.901763887Z","region_metadata_checked_at":"2026-09-18T15:41:00.901763887Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"6974fd25290feac2b0f58591c2a7cb10ae432b6e6fbafea541a7b339ee6895bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100213017273700352","occurred_at":"2026-09-16T13:19:37Z","failed_at":"e2e","signature_id":"1e90b34a567db73b55b0b0389a5bd30033829227bcd0152b9cc1ea0cbbb4ce6a","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f7e2c0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-hk9kwb8xmkfd\" to finish creating: GET https://rp.j3700352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1f5beb08-52f4-4ca9-8de7-6fbc3e2fae17\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1f5beb08-52f4-4ca9-8de7-6fbc3e2fae17\",\n \"name\": \"1f5beb08-52f4-4ca9-8de7-6fbc3e2fae17\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T13:59:17.257119371Z\",\n \"endTime\": \"2026-09-16T14:28:24.535752776Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m37s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f7e2c0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-hk9kwb8xmkfd\" to finish creating: GET https://rp.j3700352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1f5beb08-52f4-4ca9-8de7-6fbc3e2fae17\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1f5beb08-52f4-4ca9-8de7-6fbc3e2fae17\",\n \"name\": \"1f5beb08-52f4-4ca9-8de7-6fbc3e2fae17\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T13:59:17.257119371Z\",\n \"endTime\": \"2026-09-16T14:28:24.535752776Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m37s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7c3ca28d46990a466b2384b78aca46a2cf5c4315199d21bde6289ac869706c6a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100213017273700352","occurred_at":"2026-09-16T13:19:37Z","failed_at":"e2e","signature_id":"212391e614ece2195740bc51838047e93b2e7ba13e0b53f412f14fba63b492c3","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000acc200\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-hld8zl98shzl\" to finish creating: GET https://rp.j3700352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56519874-edc6-422e-930e-b7dc2d33fa54\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56519874-edc6-422e-930e-b7dc2d33fa54\",\n \"name\": \"56519874-edc6-422e-930e-b7dc2d33fa54\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T13:59:08.637456605Z\",\n \"endTime\": \"2026-09-16T14:28:14.536216515Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 20m26s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000acc200\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-hld8zl98shzl\" to finish creating: GET https://rp.j3700352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56519874-edc6-422e-930e-b7dc2d33fa54\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56519874-edc6-422e-930e-b7dc2d33fa54\",\n \"name\": \"56519874-edc6-422e-930e-b7dc2d33fa54\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T13:59:08.637456605Z\",\n \"endTime\": \"2026-09-16T14:28:14.536216515Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 20m26s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"815c95b86c6685a0130af6995e10fea9aeccbdf6727e1cb86244850735aeee8c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100213017273700352","occurred_at":"2026-09-16T13:19:37Z","failed_at":"alert","signature_id":"422af2d2e106cf074701d11ee92eac11656adc2f791fd92e942ac51d5ff857ef","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T14:14:40Z\n Ended: 2026-09-16T14:41:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3700352-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss55dasvijc9n7rk5tev7k98q8t38ut-f3o4a6n6m5t7w8i\", pod=\"router-6594467575-v5mmd\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss55dasvijc9n7rk5tev7k98q8t38ut-f3o4a6n6m5t7w8i/router-6594467575-v5mmd has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-16T14:14:40Z\n Ended: 2026-09-16T14:41:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3700352-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss55beaf4jnkm30gd64lt3c9jhq19e4-private-ingress\", pod=\"router-7db4c964-ls58f\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss55beaf4jnkm30gd64lt3c9jhq19e4-private-ingress/router-7db4c964-ls58f has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100212360202424320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6894,"pr_state":"closed","pr_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","final_merged_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T13:16:57Z","started_at":"2026-09-16T13:16:57Z","completed_at":"2026-09-16T15:12:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:23.103179176Z","timing_metadata_checked_at":"2026-09-18T15:41:23.103179176Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:23.106117987Z","region_metadata_checked_at":"2026-09-18T15:41:23.106117987Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"2010a4e7b99f4b33559cb9e9bf42587562c96cd6f17748b22e836ba9ffc44064","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100212360202424320","occurred_at":"2026-09-16T13:16:57Z","failed_at":"e2e","signature_id":"fca0d524c5ac315fe836fb88fd7ec89c3926bdf9a49356a4f7143c3576597659","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008cd0a0\u003e: \n failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-jvwfr7\" in resourcegroup=\"rg-np-skip-minor-jvwfr7-7qvvgr9wwfbq\" to finish creating: GET https://rp.j2424320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6c6ba019-61f3-4ad5-922c-02cdd95f824e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6c6ba019-61f3-4ad5-922c-02cdd95f824e\",\n \"name\": \"6c6ba019-61f3-4ad5-922c-02cdd95f824e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T14:13:13.201859753Z\",\n \"endTime\": \"2026-09-16T14:32:13.373200335Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008cd0a0\u003e: \n failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-jvwfr7\" in resourcegroup=\"rg-np-skip-minor-jvwfr7-7qvvgr9wwfbq\" to finish creating: GET https://rp.j2424320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6c6ba019-61f3-4ad5-922c-02cdd95f824e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6c6ba019-61f3-4ad5-922c-02cdd95f824e\",\n \"name\": \"6c6ba019-61f3-4ad5-922c-02cdd95f824e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T14:13:13.201859753Z\",\n \"endTime\": \"2026-09-16T14:32:13.373200335Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a746193ebcc11f6b3625af59806e73251fe757c81f8c617f710ee684b1be3458","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100212360202424320","occurred_at":"2026-09-16T13:16:57Z","failed_at":"e2e","signature_id":"fec53f2216f9f1f9da9e1788f3642b62d62882fc08fe207f753fbff13e9390bf","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00098aa40\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-6th5jd8nbg5s\" to finish creating: GET https://rp.j2424320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd1a9666-9126-4a24-9b9a-2799ae9c069b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd1a9666-9126-4a24-9b9a-2799ae9c069b\",\n \"name\": \"bd1a9666-9126-4a24-9b9a-2799ae9c069b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T13:57:52.148119457Z\",\n \"endTime\": \"2026-09-16T14:26:53.36802381Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, community-operators-catalog, hosted-cluster-config-operator, catalog-operator, redhat-operators-catalog, ingress-operator, cluster-image-registry-operator, openshift-route-controller-manager, cluster-node-tuning-operator, dns-operator, redhat-marketplace-catalog, certified-operators-catalog, cluster-network-operator, packageserver, openshift-controller-manager, cluster-policy-controller, cluster-version-operator, cluster-storage-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00098aa40\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-6th5jd8nbg5s\" to finish creating: GET https://rp.j2424320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd1a9666-9126-4a24-9b9a-2799ae9c069b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd1a9666-9126-4a24-9b9a-2799ae9c069b\",\n \"name\": \"bd1a9666-9126-4a24-9b9a-2799ae9c069b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T13:57:52.148119457Z\",\n \"endTime\": \"2026-09-16T14:26:53.36802381Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, community-operators-catalog, hosted-cluster-config-operator, catalog-operator, redhat-operators-catalog, ingress-operator, cluster-image-registry-operator, openshift-route-controller-manager, cluster-node-tuning-operator, dns-operator, redhat-marketplace-catalog, certified-operators-catalog, cluster-network-operator, packageserver, openshift-controller-manager, cluster-policy-controller, cluster-version-operator, cluster-storage-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"676e68f8a68616ac8c57dc0fe075f4a0d00c211dff572a92461d02984b57a771","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["676e68f8a68616ac8c57dc0fe075f4a0d00c211dff572a92461d02984b57a771","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100209889606373376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6886,"pr_state":"open","pr_sha":"a6aa6a03a137ec55e3f4f5d06cc620e6bf6dd7f3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T13:07:12Z","started_at":"2026-09-16T13:07:12Z","completed_at":"2026-09-16T13:46:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:31.619212347Z","timing_metadata_checked_at":"2026-09-18T15:40:31.619212347Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:31.62273016Z","region_metadata_checked_at":"2026-09-18T15:40:31.62273016Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"0bec577b0f4999e53908a53d0af6e10ba1af221ee84c23064645b944db9233f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100209889606373376","occurred_at":"2026-09-16T13:07:12Z","failed_at":"provision","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-09-16T13:31:49.803Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-09-16T13:31:51.360Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-09-16T13:31:52.622Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=6a654e03e0d8cd3ba2dcf4b7ca7ea7b9161da4e2734cccce092b3f0e303e2fe9 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6373376-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F6a654e03e0d8cd3ba2dcf4b7ca7ea7b9161da4e2734cccce092b3f0e303e2fe9\ntime=2026-09-16T13:40:24.258Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6373376-svc/providers/Microsoft.Resources/deployments/6a654e03e0d8cd3ba2dcf4b7ca7ea7b9161da4e2734cccce092b3f0e303e2fe9/operationStatuses/08584120413735528505\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6373376/providers/Microsoft.Resources/deployments/rp_cosmos_db\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6373376/providers/Microsoft.DocumentDB/databaseAccounts/arohcpci01-rp-j6373376/sqlRoleAssignments/3a943d25-7e89-52ef-bdd2-d3814561859c\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The collection with name [Resources] in provided scope [/dbs/arohcpci01-rp-j6373376/colls/Resources] could not be found.\\\\\\\\r\\\\\\\\nActivityId: 385582e2-ce0e-4904-a9f2-7469a3089fa3, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"377edc68350e3289212c2970b00ba8365c9d949a03379484101a64858ed9715c","failure_pattern":"ERROR CODE: DeploymentFailed; detail code BadRequest; detail message The collection with name [Resources] in provided scope [/dbs/\u003ccosmos-account\u003e/colls/Resources] could not be found.\\\\\\ \\\\ ActivityId: \u003cuuid\u003e, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0; provider Microsoft.DocumentDB","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["377edc68350e3289212c2970b00ba8365c9d949a03379484101a64858ed9715c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100207889619619840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6898,"pr_state":"closed","pr_sha":"3f0521ff633af0368f049bfead3093e31565b50f","final_merged_sha":"3f0521ff633af0368f049bfead3093e31565b50f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T12:59:15Z","started_at":"2026-09-16T12:59:15Z","completed_at":"2026-09-16T14:51:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:35.330462694Z","timing_metadata_checked_at":"2026-09-18T15:40:35.330462694Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:35.333870107Z","region_metadata_checked_at":"2026-09-18T15:40:35.333870107Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"169eea3d2fad49cd274868ceeed76950fa4f5e9ed8620b334a63096c1bdd5f2a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100207889619619840","occurred_at":"2026-09-16T12:59:15Z","failed_at":"e2e","signature_id":"f792cb486e1be9a347881acf7c3d2cd96a120fd226d905b4a6c6fa830ffd406b","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-jz5x7r with version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cb6700\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-jz5x7r: failed waiting for cluster=\"np-version-upgrade-cluster-jz5x7r\" in resourcegroup=\"rg-np-version-upgrade-jz5x7r-xds97j5kvxhn\" to finish creating: GET https://rp.j9619840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b59189b4-8e72-4fa2-92fe-a2a6f5dd2eb3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b59189b4-8e72-4fa2-92fe-a2a6f5dd2eb3\",\n \"name\": \"b59189b4-8e72-4fa2-92fe-a2a6f5dd2eb3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T13:41:36.451206458Z\",\n \"endTime\": \"2026-09-16T14:10:39.504401971Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-jz5x7r with version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cb6700\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-jz5x7r: failed waiting for cluster=\"np-version-upgrade-cluster-jz5x7r\" in resourcegroup=\"rg-np-version-upgrade-jz5x7r-xds97j5kvxhn\" to finish creating: GET https://rp.j9619840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b59189b4-8e72-4fa2-92fe-a2a6f5dd2eb3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b59189b4-8e72-4fa2-92fe-a2a6f5dd2eb3\",\n \"name\": \"b59189b4-8e72-4fa2-92fe-a2a6f5dd2eb3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T13:41:36.451206458Z\",\n \"endTime\": \"2026-09-16T14:10:39.504401971Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"23f134c846e33664c70b430f65ec238005ae7c12eb1c980fa3823dbae52b3720","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100207889619619840","occurred_at":"2026-09-16T12:59:15Z","failed_at":"alert","signature_id":"2c11c083dc96efa9f51e185909eda32654c395f47b2dbb78eaee76c2573a630d","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T14:03:47Z\n Ended: 2026-09-16T14:33:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9619840-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.80:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9619840/providers/microsoft.monitor/accounts/hcps-j9619840\", namespace=\"ocm-arohcpci01-2ss4t4fdubst8s5b8tltfi3bmb1s95er-u3f5u3d4c4j0m6r\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"aaf39014-4158-4621-b188-69047e816d57\"\n Description: Pod ocm-arohcpci01-2ss4t4fdubst8s5b8tltfi3bmb1s95er-u3f5u3d4c4j0m6r/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"331099401ddbd2db863114f72a8f3e7d799ec167918c723c9b5cf542d06d014a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100207889619619840","occurred_at":"2026-09-16T12:59:15Z","failed_at":"alert","signature_id":"67413ecc4fbc527d42baea24debbddc611e8d90812ae968414dd499232adf3e3","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T14:02:53Z\n Ended: 2026-09-16T14:27:49Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j9619840-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.80:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9619840/providers/microsoft.monitor/accounts/hcps-j9619840\", namespace=\"ocm-arohcpci01-2ss4t4fdubst8s5b8tltfi3bmb1s95er-u3f5u3d4c4j0m6r\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-q22d9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2ss4t4fdubst8s5b8tltfi3bmb1s95er-u3f5u3d4c4j0m6r/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-16T14:02:53Z\n Ended: 2026-09-16T14:22:48Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j9619840-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.80:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9619840/providers/microsoft.monitor/accounts/hcps-j9619840\", namespace=\"ocm-arohcpci01-2ss4t4fdubst8s5b8tltfi3bmb1s95er-u3f5u3d4c4j0m6r\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-q22d9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2ss4t4fdubst8s5b8tltfi3bmb1s95er-u3f5u3d4c4j0m6r/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"827d46c4da5c3e54e54318a48f0d6cab11411ac061d9d685c9c9d17f6d5b1232","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100207889619619840","occurred_at":"2026-09-16T12:59:15Z","failed_at":"alert","signature_id":"18b82d4d472bbc8192b3a0be8a58b9abe4cf0da273e711ee46164983b7c2c490","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T14:16:51Z\n Ended: 2026-09-16T14:34:49Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9619840-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.80:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9619840/providers/microsoft.monitor/accounts/hcps-j9619840\", namespace=\"ocm-arohcpci01-2ss4t4fdubst8s5b8tltfi3bmb1s95er-u3f5u3d4c4j0m6r\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-q22d9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ss4t4fdubst8s5b8tltfi3bmb1s95er-u3f5u3d4c4j0m6r/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"d86f92ee54bf8d545a3991029b1cd35edb3609a45a60ea8223cefaf12e63277f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100207889619619840","occurred_at":"2026-09-16T12:59:15Z","failed_at":"alert","signature_id":"aede2952cb21cd0c07ff4ac502b4109b4236290ff6f43397958c99820cf77924","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 8 time(s) (1 unknown, 7 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T13:52:47Z\n Ended: 2026-09-16T14:11:52Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9619840-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss4t6obva3desivmr8hmkd7a8qlvbhm-n2t8z7n2e0x3c5e\", pod=\"router-d9fc9d667-sk5dl\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss4t6obva3desivmr8hmkd7a8qlvbhm-n2t8z7n2e0x3c5e/router-d9fc9d667-sk5dl has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6779/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100198040831791104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6779,"pr_state":"open","pr_sha":"99221fc6038f3d6c1d9ba3816ede612d03e48973","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T12:20:07Z","started_at":"2026-09-16T12:20:07Z","completed_at":"2026-09-16T13:19:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:21.354730621Z","timing_metadata_checked_at":"2026-09-18T15:40:21.354730621Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:21.357754432Z","region_metadata_checked_at":"2026-09-18T15:40:21.357754432Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"5b6648ec44f004f45479370c6e74059ae675f25d8870b354a508488c8979bf1b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6779/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100198040831791104","occurred_at":"2026-09-16T12:20:07Z","failed_at":"provision","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","test_suite":"step graph","failure_text":"time=2026-09-16T12:42:02.218Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-16T12:42:02.867Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-16T12:42:03.803Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=f62142d1821bad4afc9418f7e325718ebc679cc0887c30afb66e545abef06520 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1791104-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff62142d1821bad4afc9418f7e325718ebc679cc0887c30afb66e545abef06520\ntime=2026-09-16T13:12:01.951Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100196783199096832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6791,"pr_state":"closed","pr_sha":"764ee60fa32d166f92c6c286f96007787ee189de","final_merged_sha":"00c5d478babb26a92f08e45ee9f4c322bafd7507","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T12:15:07Z","started_at":"2026-09-16T12:15:07Z","completed_at":"2026-09-16T14:05:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:22.891283049Z","timing_metadata_checked_at":"2026-09-18T15:40:22.891283049Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:22.894489161Z","region_metadata_checked_at":"2026-09-18T15:40:22.894489161Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"2f11e9f6845a95e6ca50ddbcf6cb8540f8bdba5755d964d3aaa5d3fcd9b544f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100196783199096832","occurred_at":"2026-09-16T12:15:07Z","failed_at":"e2e","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"9eb715fef3c45299b15d5aa27edd90c99c14f670d65742e3461017c2ee3b3792","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100196783199096832","occurred_at":"2026-09-16T12:15:07Z","failed_at":"e2e","signature_id":"d49dc00210e1aa19acac371c4b3bbf0262ff2f7023f27f607ca12332c134c5be","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013c07a0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-42qs67sfvs6r\" to finish creating: GET https://rp.j9096832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62ed0453-0f1a-4cb5-b502-4690d5fe8e28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62ed0453-0f1a-4cb5-b502-4690d5fe8e28\",\n \"name\": \"62ed0453-0f1a-4cb5-b502-4690d5fe8e28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:56:55.481514146Z\",\n \"endTime\": \"2026-09-16T13:25:58.518527402Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m38s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013c07a0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-42qs67sfvs6r\" to finish creating: GET https://rp.j9096832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62ed0453-0f1a-4cb5-b502-4690d5fe8e28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62ed0453-0f1a-4cb5-b502-4690d5fe8e28\",\n \"name\": \"62ed0453-0f1a-4cb5-b502-4690d5fe8e28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:56:55.481514146Z\",\n \"endTime\": \"2026-09-16T13:25:58.518527402Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m38s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bf32e3b21b5012fe86be110cf3ff526355836b77d3940800a6d2b9bf0228b8d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100196783199096832","occurred_at":"2026-09-16T12:15:07Z","failed_at":"e2e","signature_id":"c6e0c1f0e5cc2283be488e36d90c53cb7425625d072448ae938150a9f6555ec0","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-d52gscxjdr6c/cluster-candidate-4-21-wc8v7p should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f7c4c0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-wc8v7p: failed waiting for cluster=\"cluster-candidate-4-21-wc8v7p\" in resourcegroup=\"rg-candidate-4-21-d52gscxjdr6c\" to finish creating: GET https://rp.j9096832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ecd643be-f810-4322-9880-6c4747c1f655\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ecd643be-f810-4322-9880-6c4747c1f655\",\n \"name\": \"ecd643be-f810-4322-9880-6c4747c1f655\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:56:57.32608726Z\",\n \"endTime\": \"2026-09-16T13:25:58.487825893Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m4s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-d52gscxjdr6c/cluster-candidate-4-21-wc8v7p should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f7c4c0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-wc8v7p: failed waiting for cluster=\"cluster-candidate-4-21-wc8v7p\" in resourcegroup=\"rg-candidate-4-21-d52gscxjdr6c\" to finish creating: GET https://rp.j9096832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ecd643be-f810-4322-9880-6c4747c1f655\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ecd643be-f810-4322-9880-6c4747c1f655\",\n \"name\": \"ecd643be-f810-4322-9880-6c4747c1f655\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:56:57.32608726Z\",\n \"endTime\": \"2026-09-16T13:25:58.487825893Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m4s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194885893099520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6439,"pr_state":"open","pr_sha":"b6b00cdc8ea3ee249a9faa811955e4eb095652ef","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T12:07:35Z","started_at":"2026-09-16T12:07:35Z","completed_at":"2026-09-16T13:07:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:06.203889099Z","timing_metadata_checked_at":"2026-09-18T15:40:06.203889099Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:06.207860013Z","region_metadata_checked_at":"2026-09-18T15:40:06.207860013Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"2b31128fffe954a54427dd4686c90d57da5a8f8989ad4e9fe90c8d739533d770","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194885893099520","occurred_at":"2026-09-16T12:07:35Z","failed_at":"provision","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","test_suite":"step graph","failure_text":"time=2026-09-16T12:31:03.652Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-16T12:31:04.249Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-16T12:31:05.229Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=c3798ec18923b6e7d63ee7828445bbe6248d8a37c22e598b697019726ef5af5a portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3099520-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fc3798ec18923b6e7d63ee7828445bbe6248d8a37c22e598b697019726ef5af5a\ntime=2026-09-16T13:01:03.275Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194205623128064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6672,"pr_state":"closed","pr_sha":"6a4134596093a91ac8b3067ec4d6ccf2dfe9ce43","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T12:04:52Z","started_at":"2026-09-16T12:04:52Z","completed_at":"2026-09-16T14:10:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:10.511047808Z","timing_metadata_checked_at":"2026-09-18T15:40:10.511047808Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:10.514248619Z","region_metadata_checked_at":"2026-09-18T15:40:10.514248619Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"0d013dd27cb92e192e91a6f73accb41d2b77623a89ff691a7b9bd9ac8042e5f5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194205623128064","occurred_at":"2026-09-16T12:04:52Z","failed_at":"e2e","signature_id":"9b7d539624448efd4996e6b535e22c740df41bfa52b50eab26733ce795540f15","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.000s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0012e60a0\u003e: \n failed to get kusto cluster name from config: configuration: key kusto not found\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.000s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0012e60a0\u003e: \n failed to get kusto cluster name from config: configuration: key kusto not found\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c71ebc2a93d9fc10a75dcaf39a2b1325de6379c3891238b64d1e4e8f4281df9a","failure_pattern":"failed to get kusto cluster \u003ckusto-cluster\u003e from config: configuration: key kusto not found","search_query":"failed to get kusto cluster name from config: configuration: key kusto not found"}},{"row_id":"194198ddc22c31dfd60c0ede0c86588af591dc741659d0156e091095fe80379f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194205623128064","occurred_at":"2026-09-16T12:04:52Z","failed_at":"e2e","signature_id":"1d95e250094820742e5d106c9a6198395eff4115c52cfbf3176662d0996cbb8f","test_name":"KSM HCP Metrics kube_node_info metrics should be present in Azure Monitor for the happy-path cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:74]: failed to look up HCP Prometheus endpoint\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00072e820\u003e: \n failed to get workspace hcps-j3128064: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3128064/providers/Microsoft.Monitor/accounts/hcps-j3128064\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceGroupNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-j3128064' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:74]: failed to look up HCP Prometheus endpoint\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00072e820\u003e: \n failed to get workspace hcps-j3128064: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3128064/providers/Microsoft.Monitor/accounts/hcps-j3128064\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceGroupNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-j3128064' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"575b0de6701cb1cdcdd3b5c4e6523248471d3aa67c3c0c1fed243233e4014b33","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.Monitor","search_query":"ERROR CODE: ResourceGroupNotFound"}},{"row_id":"41764aa5b75af1a8ee967023eae306f95da2fc5f0066816fdbf68caf526ae13c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194205623128064","occurred_at":"2026-09-16T12:04:52Z","failed_at":"e2e","signature_id":"2300bcf0e83155e35375beb6ab2cd2dd1e60df39d954749c7fd210ed41e13cde","test_name":"AMA Metrics service metrics from SVC and MGMT clusters should be present in Azure Monitor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/ama_metrics.go:83]: failed to look up SVC Prometheus endpoint for workspace services-j3128064 in resource group hcp-underlay-ci01-j3128064\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010489e0\u003e: \n failed to get workspace services-j3128064: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3128064/providers/Microsoft.Monitor/accounts/services-j3128064\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceGroupNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-j3128064' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/ama_metrics.go:83]: failed to look up SVC Prometheus endpoint for workspace services-j3128064 in resource group hcp-underlay-ci01-j3128064\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010489e0\u003e: \n failed to get workspace services-j3128064: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3128064/providers/Microsoft.Monitor/accounts/services-j3128064\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceGroupNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-j3128064' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"575b0de6701cb1cdcdd3b5c4e6523248471d3aa67c3c0c1fed243233e4014b33","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.Monitor","search_query":"ERROR CODE: ResourceGroupNotFound"}},{"row_id":"5f4c6bf59c8bef6de9cda97b4ca9227407ff62faebcb702c84346dbf934f66f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194205623128064","occurred_at":"2026-09-16T12:04:52Z","failed_at":"e2e","signature_id":"e12e1b463aa599590cba976eac3785b82a07ad6fd298aa2e1664ebd292e43daa","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.007s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0017cdba0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:34702-\u003e172.30.0.10:53: read: connection refused\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.007s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0017cdba0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:34702-\u003e172.30.0.10:53: read: connection refused\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02"}},{"row_id":"60d07f61630e768a0a86e4ddc334a5ef21bf2f91ac45d55b97c5d9163ceea194","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194205623128064","occurred_at":"2026-09-16T12:04:52Z","failed_at":"alert","signature_id":"3a6cb6146cbf8c095d871f7807f59ec635628a6cc72ba515bf71641eb6ad5bd1","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 44 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T13:01:29Z\n Ended: 2026-09-16T13:27:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss457poa72ja0btdj1ld3p1ldvs4gtt-b7a5e9g4f3j4n2u\", pod=\"kube-state-metrics-hcp-8654bffc85-smj2c\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss457poa72ja0btdj1ld3p1ldvs4gtt-b7a5e9g4f3j4n2u/kube-state-metrics-hcp-8654bffc85-smj2c has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-16T13:02:31Z\n Ended: 2026-09-16T13:33:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45nv36i1dc0a9pkmr8als6f6o3fig-l7l2b5f0n2d0s0z\", pod=\"kube-state-metrics-hcp-8654bffc85-s6k2r\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45nv36i1dc0a9pkmr8als6f6o3fig-l7l2b5f0n2d0s0z/kube-state-metrics-hcp-8654bffc85-s6k2r has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-16T13:02:31Z\n Ended: 2026-09-16T13:32:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45a9ct6a0mh48hpkkmm7h44ov42vt-m4m6l3b4q9e9f9l\", pod=\"kube-state-metrics-hcp-8654bffc85-6czgh\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45a9ct6a0mh48hpkkmm7h44ov42vt-m4m6l3b4q9e9f9l/kube-state-metrics-hcp-8654bffc85-6czgh has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-16T13:02:31Z\n Ended: 2026-09-16T13:20:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45daaanj7b9tf929bmahf1ptl7bk8-oidc-wi-cluster\", pod=\"kube-state-metrics-hcp-8654bffc85-kglmw\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45daaanj7b9tf929bmahf1ptl7bk8-oidc-wi-cluster/kube-state-metrics-hcp-8654bffc85-kglmw has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-16T13:02:31Z\n Ended: 2026-09-16T13:17:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45e9nd54k58re0kpbnm8ce4j9mk0j-sre-hcp-cluster\", pod=\"kube-state-metrics-hcp-8654bffc85-d5xl8\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45e9nd54k58re0kpbnm8ce4j9mk0j-sre-hcp-cluster/kube-state-metrics-hcp-8654bffc85-d5xl8 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-16T13:02:31Z\n Ended: 2026-09-16T13:20:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss455n64tgfov79qogjllrl433lkcdu-a4n5t3g6x1e8t8w\", pod=\"kube-state-metrics-hcp-8654bffc85-l76th\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss455n64tgfov79qogjllrl433lkcdu-a4n5t3g6x1e8t8w/kube-state-metrics-hcp-8654bffc85-l76th has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-16T13:02:31Z\n Ended: 2026-09-16T13:11:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss457qlr01ukp4lj3sk5hot62sn7k8r-u3b8i6t7f6q2z8w\", pod=\"kube-state-metrics-hcp-8654bffc85-prmvm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss457qlr01ukp4lj3sk5hot62sn7k8r-u3b8i6t7f6q2z8w/kube-state-metrics-hcp-8654bffc85-prmvm has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-16T13:02:31Z\n Ended: 2026-09-16T13:29:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45aebd49uls1uivkklt2gp3aq0a75-p4k0j6h6l2a3p5k\", pod=\"kube-state-metrics-hcp-8654bffc85-ptg8g\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45aebd49uls1uivkklt2gp3aq0a75-p4k0j6h6l2a3p5k/kube-state-metrics-hcp-8654bffc85-ptg8g has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-16T13:02:31Z\n Ended: 2026-09-16T13:30:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45d6n0244mp2oimtij4qn88f83qjc-l2o2i1p1b9i7k1b\", pod=\"kube-state-metrics-hcp-8654bffc85-vs7hk\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45d6n0244mp2oimtij4qn88f83qjc-l2o2i1p1b9i7k1b/kube-state-metrics-hcp-8654bffc85-vs7hk has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-16T13:02:31Z\n Ended: 2026-09-16T13:21:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss453dqari5t3de9k8dotol85jfinfd-q0j6a1e9b4w7e1c\", pod=\"kube-state-metrics-hcp-8654bffc85-2b4bl\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss453dqari5t3de9k8dotol85jfinfd-q0j6a1e9b4w7e1c/kube-state-metrics-hcp-8654bffc85-2b4bl has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-16T13:03:28Z\n Ended: 2026-09-16T13:41:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45ahkai3d0q9fcpsr6mjdaf7nr6ha-o2t8d3y0l7d1x7n\", pod=\"kube-state-metrics-hcp-8654bffc85-6v5ws\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45ahkai3d0q9fcpsr6mjdaf7nr6ha-o2t8d3y0l7d1x7n/kube-state-metrics-hcp-8654bffc85-6v5ws has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-16T13:03:28Z\n Ended: 2026-09-16T13:43:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45amlhvl451enbef5jfjbe02g63a8-f1q4q6k8n9u8m3k\", pod=\"kube-state-metrics-hcp-8654bffc85-8r7bj\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45amlhvl451enbef5jfjbe02g63a8-f1q4q6k8n9u8m3k/kube-state-metrics-hcp-8654bffc85-8r7bj has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-09-16T13:03:28Z\n Ended: 2026-09-16T13:32:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45d2ps5pdtsqgj43r9su0i07fh1h8-m2y3w9f0u0r9k3l\", pod=\"kube-state-metrics-hcp-8654bffc85-hzvqb\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45d2ps5pdtsqgj43r9su0i07fh1h8-m2y3w9f0u0r9k3l/kube-state-metrics-hcp-8654bffc85-hzvqb has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-09-16T13:03:28Z\n Ended: 2026-09-16T13:21:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45avj4f66o61uit0kcksueujolvbq-k0e8m9j1h2j9u0i\", pod=\"router-7f899b48d7-n6wr7\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45avj4f66o61uit0kcksueujolvbq-k0e8m9j1h2j9u0i/router-7f899b48d7-n6wr7 has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-09-16T13:03:29Z\n Ended: 2026-09-16T13:17:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45dhik6j6sb9fbaaodapkvcpnjfmt-f4z1z7c9x4w9k3g\", pod=\"kube-state-metrics-hcp-8654bffc85-wbs5w\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45dhik6j6sb9fbaaodapkvcpnjfmt-f4z1z7c9x4w9k3g/kube-state-metrics-hcp-8654bffc85-wbs5w has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-09-16T13:03:29Z\n Ended: 2026-09-16T13:31:27Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45a0lv485th7ov853bmhqda7d1grb-f5l5n5v0d8y2u6d\", pod=\"kube-state-metrics-hcp-8654bffc85-mpj29\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45a0lv485th7ov853bmhqda7d1grb-f5l5n5v0d8y2u6d/kube-state-metrics-hcp-8654bffc85-mpj29 has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-09-16T13:03:29Z\n Ended: 2026-09-16T13:22:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45aneg2j2o4qth6flecfajj5tvki4-l2o8u6v9c5n7n6q\", pod=\"kube-state-metrics-hcp-8654bffc85-pc8f8\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45aneg2j2o4qth6flecfajj5tvki4-l2o8u6v9c5n7n6q/kube-state-metrics-hcp-8654bffc85-pc8f8 has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-09-16T13:03:29Z\n Ended: 2026-09-16T13:41:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45aj504lvh1njbod15kk0htfcugdd-w0e0y5e7r4m3o8g\", pod=\"kube-state-metrics-hcp-8654bffc85-smbng\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45aj504lvh1njbod15kk0htfcugdd-w0e0y5e7r4m3o8g/kube-state-metrics-hcp-8654bffc85-smbng has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-09-16T13:03:29Z\n Ended: 2026-09-16T13:41:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45alqfdjojoc6uvrrquvn69ep86hp-p4l3h4i3y1n7n9p\", pod=\"kube-state-metrics-hcp-8654bffc85-kts9j\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45alqfdjojoc6uvrrquvn69ep86hp-p4l3h4i3y1n7n9p/kube-state-metrics-hcp-8654bffc85-kts9j has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-09-16T13:03:29Z\n Ended: 2026-09-16T13:17:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45isafobnnvse3kq39umk693uve6h-r2t0l1d7c7a9d1e\", pod=\"kube-state-metrics-hcp-8654bffc85-mvs9w\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45isafobnnvse3kq39umk693uve6h-r2t0l1d7c7a9d1e/kube-state-metrics-hcp-8654bffc85-mvs9w has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-09-16T13:03:29Z\n Ended: 2026-09-16T13:27:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45d23ufser4jmn2hba7t0vm2f38rb-g2m8b0v1u1t7v8f\", pod=\"kube-state-metrics-hcp-8654bffc85-gtx4b\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45d23ufser4jmn2hba7t0vm2f38rb-g2m8b0v1u1t7v8f/kube-state-metrics-hcp-8654bffc85-gtx4b has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-09-16T13:03:29Z\n Ended: 2026-09-16T13:29:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45ai5u5hobiuu876l5gr5krkhf59n-x0a0u8j1u0z8t2z\", pod=\"kube-state-metrics-hcp-8654bffc85-8xwn8\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45ai5u5hobiuu876l5gr5krkhf59n-x0a0u8j1u0z8t2z/kube-state-metrics-hcp-8654bffc85-8xwn8 has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-09-16T13:03:29Z\n Ended: 2026-09-16T13:39:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45am3qd3i5mseg9vp5t7veg60i9u8-f9b4r1g2v3v3y5u\", pod=\"kube-state-metrics-hcp-8654bffc85-plqmq\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45am3qd3i5mseg9vp5t7veg60i9u8-f9b4r1g2v3v3y5u/kube-state-metrics-hcp-8654bffc85-plqmq has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-09-16T13:03:29Z\n Ended: 2026-09-16T13:11:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45cq3i9jdtaiu413fjc4tb5c4se9l-basic-cluster\", pod=\"kube-state-metrics-hcp-8654bffc85-zm5pl\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45cq3i9jdtaiu413fjc4tb5c4se9l-basic-cluster/kube-state-metrics-hcp-8654bffc85-zm5pl has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-09-16T13:03:29Z\n Ended: 2026-09-16T13:43:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45dik9ueoegvhtmnvsnteoq0g4du1-n7b6v4i2o7m1u0u\", pod=\"kube-state-metrics-hcp-8654bffc85-slk75\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45dik9ueoegvhtmnvsnteoq0g4du1-n7b6v4i2o7m1u0u/kube-state-metrics-hcp-8654bffc85-slk75 has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-09-16T13:03:29Z\n Ended: 2026-09-16T13:22:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45aknhr7bgqono4l6o0gk5trcumoj-b3b9b1i0u4d3u6y\", pod=\"kube-state-metrics-hcp-8654bffc85-hqwvp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45aknhr7bgqono4l6o0gk5trcumoj-b3b9b1i0u4d3u6y/kube-state-metrics-hcp-8654bffc85-hqwvp has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-09-16T13:03:29Z\n Ended: 2026-09-16T13:29:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss453dfmfp3h1a6k0tbejjbctsua7j9-k1i2q3i9i2w2j2x\", pod=\"kube-state-metrics-hcp-8654bffc85-2jmxg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss453dfmfp3h1a6k0tbejjbctsua7j9-k1i2q3i9i2w2j2x/kube-state-metrics-hcp-8654bffc85-2jmxg has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-09-16T13:03:29Z\n Ended: 2026-09-16T13:34:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45djm06hu474k4gire3e7l44olc34-b3g5s1b1p5g3w0y\", pod=\"kube-state-metrics-hcp-8654bffc85-w7q5h\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45djm06hu474k4gire3e7l44olc34-b3g5s1b1p5g3w0y/kube-state-metrics-hcp-8654bffc85-w7q5h has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-09-16T13:03:29Z\n Ended: 2026-09-16T13:23:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45d4v7s5h34gf677lbu6stjtqtp5c-g5x8y2b6l9p8d1e\", pod=\"kube-state-metrics-hcp-8654bffc85-gkwrf\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45d4v7s5h34gf677lbu6stjtqtp5c-g5x8y2b6l9p8d1e/kube-state-metrics-hcp-8654bffc85-gkwrf has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-09-16T13:03:29Z\n Ended: 2026-09-16T13:24:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45alt0da9lepo6tu2pvh5u7s687dp-ephemeral-disk\", pod=\"kube-state-metrics-hcp-8654bffc85-xcpj5\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45alt0da9lepo6tu2pvh5u7s687dp-ephemeral-disk/kube-state-metrics-hcp-8654bffc85-xcpj5 has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Resolved\n Started: 2026-09-16T13:03:29Z\n Ended: 2026-09-16T13:21:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45bc09v53lfjcmj3g54u420ob77s8-r0l2r4e5i1s6l2h\", pod=\"router-69b5485d74-72t5c\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45bc09v53lfjcmj3g54u420ob77s8-r0l2r4e5i1s6l2h/router-69b5485d74-72t5c has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Resolved\n Started: 2026-09-16T13:03:29Z\n Ended: 2026-09-16T13:22:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45d7dagj2q4fc95vdlgrbjcrkagmi-k5b4r3j2v6g5e7k\", pod=\"kube-state-metrics-hcp-8654bffc85-wxmr4\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45d7dagj2q4fc95vdlgrbjcrkagmi-k5b4r3j2v6g5e7k/kube-state-metrics-hcp-8654bffc85-wxmr4 has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Resolved\n Started: 2026-09-16T13:04:30Z\n Ended: 2026-09-16T13:19:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss455r3f3n7sks7s8o3chfas3qcq56n-ingress-cert-le\", pod=\"router-764784758f-gqndx\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss455r3f3n7sks7s8o3chfas3qcq56n-ingress-cert-le/router-764784758f-gqndx has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Resolved\n Started: 2026-09-16T13:04:30Z\n Ended: 2026-09-16T13:21:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45c06aho04tppkno3kefcqn1a2sjs-ea-listrtdzxt\", pod=\"kube-state-metrics-hcp-8654bffc85-fds47\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45c06aho04tppkno3kefcqn1a2sjs-ea-listrtdzxt/kube-state-metrics-hcp-8654bffc85-fds47 has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Resolved\n Started: 2026-09-16T13:04:30Z\n Ended: 2026-09-16T13:24:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45c5gu0g5rk3sue62gv6srpt0sv19-f9y3n5c1f1h3h1e\", pod=\"kube-state-metrics-hcp-8654bffc85-s96gm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45c5gu0g5rk3sue62gv6srpt0sv19-f9y3n5c1f1h3h1e/kube-state-metrics-hcp-8654bffc85-s96gm has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Resolved\n Started: 2026-09-16T13:04:30Z\n Ended: 2026-09-16T13:31:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45al5vb34g2kbgdpjjtt49tus9qu7-cilium-cl\", pod=\"kube-state-metrics-hcp-8654bffc85-7m95x\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45al5vb34g2kbgdpjjtt49tus9qu7-cilium-cl/kube-state-metrics-hcp-8654bffc85-7m95x has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Resolved\n Started: 2026-09-16T13:04:30Z\n Ended: 2026-09-16T13:38:27Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45d5p3kjg2dkomkt1pvkud0d8fint-k5n3r3f4t4j4d9f\", pod=\"kube-state-metrics-hcp-8654bffc85-5zf8t\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45d5p3kjg2dkomkt1pvkud0d8fint-k5n3r3f4t4j4d9f/kube-state-metrics-hcp-8654bffc85-5zf8t has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n State: Resolved\n Started: 2026-09-16T13:04:30Z\n Ended: 2026-09-16T13:26:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45au25d8jlpb2kbjloe06frtlt4u1-g4f9f6d7f1k2n3j\", pod=\"kube-state-metrics-hcp-8654bffc85-2w2pn\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45au25d8jlpb2kbjloe06frtlt4u1-g4f9f6d7f1k2n3j/kube-state-metrics-hcp-8654bffc85-2w2pn has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n State: Resolved\n Started: 2026-09-16T13:04:30Z\n Ended: 2026-09-16T13:20:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss463phv434m12h3cgfclekieomu7eo-cilium-cluster\", pod=\"router-67b67d74d4-4d6mj\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss463phv434m12h3cgfclekieomu7eo-cilium-cluster/router-67b67d74d4-4d6mj has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n State: Resolved\n Started: 2026-09-16T13:04:30Z\n Ended: 2026-09-16T13:24:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45d0pa1sd8jpnk5mmr7bmpmc1uh27-d9u9b5p5t5x3a7o\", pod=\"kube-state-metrics-hcp-8654bffc85-jzd7x\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45d0pa1sd8jpnk5mmr7bmpmc1uh27-d9u9b5p5t5x3a7o/kube-state-metrics-hcp-8654bffc85-jzd7x has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n State: Resolved\n Started: 2026-09-16T13:04:30Z\n Ended: 2026-09-16T13:44:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45enm2q1mrr0ikcmo5so09hpk1440-g7s2v6e1t4t3v3w\", pod=\"kube-state-metrics-hcp-8654bffc85-strhx\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45enm2q1mrr0ikcmo5so09hpk1440-g7s2v6e1t4t3v3w/kube-state-metrics-hcp-8654bffc85-strhx has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n State: Resolved\n Started: 2026-09-16T13:04:30Z\n Ended: 2026-09-16T13:17:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45alef9infcda8pp6lj56gnqiqkt1-s2m0d6m5z5x1p1o\", pod=\"kube-state-metrics-hcp-8654bffc85-rlzdp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45alef9infcda8pp6lj56gnqiqkt1-s2m0d6m5z5x1p1o/kube-state-metrics-hcp-8654bffc85-rlzdp has been in a non-ready state for longer than 5 minutes.\nFiring 43:\n State: Resolved\n Started: 2026-09-16T13:04:30Z\n Ended: 2026-09-16T13:12:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss45ithac2bg4g097vh3s95jf7bpkvs-s3q1o0z3y5x3y8v\", pod=\"kube-state-metrics-hcp-8654bffc85-rj8kx\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss45ithac2bg4g097vh3s95jf7bpkvs-s3q1o0z3y5x3y8v/kube-state-metrics-hcp-8654bffc85-rj8kx has been in a non-ready state for longer than 5 minutes.\nFiring 44:\n State: Resolved\n Started: 2026-09-16T13:04:30Z\n Ended: 2026-09-16T13:29:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3128064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss457tpj3vtp5gmo526i67mlaqoerts-ea-cluster\", pod=\"kube-state-metrics-hcp-8654bffc85-4cgrf\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss457tpj3vtp5gmo526i67mlaqoerts-ea-cluster/kube-state-metrics-hcp-8654bffc85-4cgrf has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["575b0de6701cb1cdcdd3b5c4e6523248471d3aa67c3c0c1fed243233e4014b33","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","c71ebc2a93d9fc10a75dcaf39a2b1325de6379c3891238b64d1e4e8f4281df9a"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194111955931136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6977,"pr_state":"closed","pr_sha":"557c04379e033b9de0bd85904b49103f2cdef1b1","final_merged_sha":"557c04379e033b9de0bd85904b49103f2cdef1b1","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T12:04:19Z","started_at":"2026-09-16T12:04:19Z","completed_at":"2026-09-16T13:06:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:05.623168389Z","timing_metadata_checked_at":"2026-09-18T15:41:05.623168389Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:05.629278911Z","region_metadata_checked_at":"2026-09-18T15:41:05.629278911Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"7df47dcdb59f881760b5406878218f7628ec22198d63e57860199bc2ea6f3673","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194111955931136","occurred_at":"2026-09-16T12:04:19Z","failed_at":"provision","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","test_suite":"step graph","failure_text":"time=2026-09-16T12:29:06.362Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-16T12:29:07.004Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-16T12:29:08.123Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=7f3003552f349523d7e6c238bf9cccd3058ec60bc5ec75c8955458c50e5a5797 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5931136-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F7f3003552f349523d7e6c238bf9cccd3058ec60bc5ec75c8955458c50e5a5797\ntime=2026-09-16T12:59:06.018Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100192568775020544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6964,"pr_state":"closed","pr_sha":"af707ce72df7d034eaee96beafbefdfe37fc0800","final_merged_sha":"af707ce72df7d034eaee96beafbefdfe37fc0800","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T11:58:22Z","started_at":"2026-09-16T11:58:22Z","completed_at":"2026-09-16T12:38:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:58.616873268Z","timing_metadata_checked_at":"2026-09-18T15:40:58.616873268Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:58.62006248Z","region_metadata_checked_at":"2026-09-18T15:40:58.62006248Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"9166a331c9691e16339cf2edfa1c704b9c97f7cd1397deea9c3bd8d87dad3112","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100192568775020544","occurred_at":"2026-09-16T11:58:22Z","failed_at":"provision","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-09-16T12:27:55.094Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-09-16T12:27:56.479Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-09-16T12:27:57.833Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=b489aeac3f7b0efc7498443b42a85f2f65ee62ab4f82355239f486d3dbc78483 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5020544-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fb489aeac3f7b0efc7498443b42a85f2f65ee62ab4f82355239f486d3dbc78483\ntime=2026-09-16T12:31:28.399Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5020544-svc/providers/Microsoft.Resources/deployments/b489aeac3f7b0efc7498443b42a85f2f65ee62ab4f82355239f486d3dbc78483/operationStatuses/08584120452084424414\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5020544-svc/providers/Microsoft.Resources/deployments/cluster-6sk23df5akobi\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5020544-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j5020544-svc\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"GetMSICredentialError\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Get managed identity credential failed. Details: Category: ClientError; Code: NotFound; SubCode: NotFound; Message: unexpected response from MSI data plane, length of returned certificate: 0; Dependency: MSI Credential; AKSTeam: RP\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6bab59ff05e72a771d612966dd1e7024c1a3d21427661ca54c8003eb2c7e563","failure_pattern":"ERROR CODE: DeploymentFailed; detail code GetMSICredentialError; detail message Get managed identity credential failed. Details: Category: ClientError; Code: NotFound; SubCode: NotFound; Message: unexpected response from MSI data plane, length of returned certificate: 0; Dependency: MSI Credential; AKSTeam: RP; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a6bab59ff05e72a771d612966dd1e7024c1a3d21427661ca54c8003eb2c7e563"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100192526756483072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5846,"pr_state":"closed","pr_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","final_merged_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T11:58:12Z","started_at":"2026-09-16T11:58:12Z","completed_at":"2026-09-16T13:05:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:04.054299461Z","timing_metadata_checked_at":"2026-09-18T15:40:04.054299461Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:04.057683073Z","region_metadata_checked_at":"2026-09-18T15:40:04.057683073Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"aa2828be527a477845b275a813a7e8f22cb5af30c6a39fc0055850fbf09ef563","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100192526756483072","occurred_at":"2026-09-16T11:58:12Z","failed_at":"provision","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","test_suite":"step graph","failure_text":"time=2026-09-16T12:29:10.225Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-16T12:29:10.864Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-09-16T12:29:11.770Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=1b97e8958d39ef4f7e29556f7815dac2c009b4e6ab514d8c7781025dc9895978 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6483072-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F1b97e8958d39ef4f7e29556f7815dac2c009b4e6ab514d8c7781025dc9895978\ntime=2026-09-16T12:59:09.868Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6971/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100192488521207808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6971,"pr_state":"closed","pr_sha":"ea9e0ef11ac9b22da4762ee1b3c145bb7eee1a05","final_merged_sha":"ea9e0ef11ac9b22da4762ee1b3c145bb7eee1a05","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-16T11:58:03Z","started_at":"2026-09-16T11:58:03Z","completed_at":"2026-09-16T14:13:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:02.242695616Z","timing_metadata_checked_at":"2026-09-18T15:41:02.242695616Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:02.245699027Z","region_metadata_checked_at":"2026-09-18T15:41:02.245699027Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100190163022909440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6974,"pr_state":"closed","pr_sha":"ad18c381b0fe07c50afb22f07e5b849b9d525738","final_merged_sha":"fdb1b526d51e98ec4ae2b5a2fc9fae30dbe31d6d","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T11:48:49Z","started_at":"2026-09-16T11:48:49Z","completed_at":"2026-09-16T13:05:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:03.62124468Z","timing_metadata_checked_at":"2026-09-18T15:41:03.62124468Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:03.624888493Z","region_metadata_checked_at":"2026-09-18T15:41:03.624888493Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"40c5542da7b66634868433abedad9753cb09534d6b4f429947783297fd255fa9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100190163022909440","occurred_at":"2026-09-16T11:48:49Z","failed_at":"provision","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","test_suite":"step graph","failure_text":"time=2026-09-16T12:28:37.195Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-16T12:28:38.137Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-16T12:28:39.285Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=a97919f9baaee18d96914ec7a53b966faabe4a5aaaf7dd5f49d556a1afbdb617 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2909440-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fa97919f9baaee18d96914ec7a53b966faabe4a5aaaf7dd5f49d556a1afbdb617\ntime=2026-09-16T12:58:36.820Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189557025673216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6976,"pr_state":"open","pr_sha":"3f37770fba22a55b4336e3a85cbf275bf37c8279","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T11:46:24Z","started_at":"2026-09-16T11:46:24Z","completed_at":"2026-09-16T13:53:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:04.557382751Z","timing_metadata_checked_at":"2026-09-18T15:41:04.557382751Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:04.560745163Z","region_metadata_checked_at":"2026-09-18T15:41:04.560745163Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"4b7af25bbd5253cafb2ed1adae90a98eabaeb4f790490de781dbfa8e80426d86","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189557025673216","occurred_at":"2026-09-16T11:46:24Z","failed_at":"e2e","signature_id":"4c23de01cdd9b886d00d4c9a0959f33955c73d1a556a45314d353da1da388706","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009a0c20\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-xcwwm98r852j\" to finish creating: GET https://rp.j5673216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9e7e240-22cf-4186-a0b5-ed5964184230\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9e7e240-22cf-4186-a0b5-ed5964184230\",\n \"name\": \"c9e7e240-22cf-4186-a0b5-ed5964184230\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:39:41.842014119Z\",\n \"endTime\": \"2026-09-16T13:08:44.867300406Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009a0c20\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-xcwwm98r852j\" to finish creating: GET https://rp.j5673216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9e7e240-22cf-4186-a0b5-ed5964184230\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9e7e240-22cf-4186-a0b5-ed5964184230\",\n \"name\": \"c9e7e240-22cf-4186-a0b5-ed5964184230\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:39:41.842014119Z\",\n \"endTime\": \"2026-09-16T13:08:44.867300406Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b414f524aec7c4ea3c6d464450b9af6f734740715a61aff1c3a8f53547f62641","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5c8d65723c49f1f7b7c9fa36b849c224a392732c17068b8d133f4a6ee6dbcd37","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189557025673216","occurred_at":"2026-09-16T11:46:24Z","failed_at":"alert","signature_id":"7451ba3db591daddff6044c850b50a806f031b44b26c0a9cb44f8b82e28dd244","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T12:51:57Z\n Ended: 2026-09-16T13:20:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5673216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss402jtj3okm3l0itnvu8lp11ckp79a-v20261001\", pod=\"router-6c568fff88-cgf48\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss402jtj3okm3l0itnvu8lp11ckp79a-v20261001/router-6c568fff88-cgf48 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"8b93ddf98b1adab8bb4a73a870a185da883158b4e4b8e007bf4bb0e8ba1eb0df","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189557025673216","occurred_at":"2026-09-16T11:46:24Z","failed_at":"e2e","signature_id":"5d3daeaf2541170cf2048425db2ad55ef8977c82724f199e4513a26eb43628d7","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-7k4hphh5btnd/cluster-candidate-4-20-f2v4f7 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cf6760\u003e: \n failed to create HCP cluster cluster-candidate-4-20-f2v4f7: failed waiting for cluster=\"cluster-candidate-4-20-f2v4f7\" in resourcegroup=\"rg-candidate-4-20-7k4hphh5btnd\" to finish creating: GET https://rp.j5673216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af3ecd66-e6f0-4eb3-95a7-f5a158757dbd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af3ecd66-e6f0-4eb3-95a7-f5a158757dbd\",\n \"name\": \"af3ecd66-e6f0-4eb3-95a7-f5a158757dbd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:40:01.27881814Z\",\n \"endTime\": \"2026-09-16T13:09:04.874482625Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.38 is Partial (want Completed), started 20m32s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-7k4hphh5btnd/cluster-candidate-4-20-f2v4f7 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cf6760\u003e: \n failed to create HCP cluster cluster-candidate-4-20-f2v4f7: failed waiting for cluster=\"cluster-candidate-4-20-f2v4f7\" in resourcegroup=\"rg-candidate-4-20-7k4hphh5btnd\" to finish creating: GET https://rp.j5673216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af3ecd66-e6f0-4eb3-95a7-f5a158757dbd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af3ecd66-e6f0-4eb3-95a7-f5a158757dbd\",\n \"name\": \"af3ecd66-e6f0-4eb3-95a7-f5a158757dbd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:40:01.27881814Z\",\n \"endTime\": \"2026-09-16T13:09:04.874482625Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.38 is Partial (want Completed), started 20m32s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9dd55c2383f30a788c6ff53ec1a399fafaaca242ce9a398251b077a9e540bab4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189557025673216","occurred_at":"2026-09-16T11:46:24Z","failed_at":"e2e","signature_id":"cedab3b9f607c3ec7848c570f240cf2b35352afbc688b78a3a96a96f564deb7f","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010fab40\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-dztrtj585gtc\" to finish creating: GET https://rp.j5673216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e071038e-8e42-4a4f-90ab-1d7962337b4e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e071038e-8e42-4a4f-90ab-1d7962337b4e\",\n \"name\": \"e071038e-8e42-4a4f-90ab-1d7962337b4e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:41:22.959566801Z\",\n \"endTime\": \"2026-09-16T13:10:24.832743272Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m16s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010fab40\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-dztrtj585gtc\" to finish creating: GET https://rp.j5673216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e071038e-8e42-4a4f-90ab-1d7962337b4e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e071038e-8e42-4a4f-90ab-1d7962337b4e\",\n \"name\": \"e071038e-8e42-4a4f-90ab-1d7962337b4e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:41:22.959566801Z\",\n \"endTime\": \"2026-09-16T13:10:24.832743272Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m16s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c0f592d1c4f78a9db25a726fdc4380ac09bb1f18d5b33b827e317e9c4dfcfa89","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189557025673216","occurred_at":"2026-09-16T11:46:24Z","failed_at":"e2e","signature_id":"3bf8a07ef9de6057d5469a6d5677df4989f7484e06e3a81c4e3df9504d3c4305","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:98]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c10220\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-89n7j94drhpq\" to finish creating: GET https://rp.j5673216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/494c647c-02a4-4a85-8468-6289598342b3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/494c647c-02a4-4a85-8468-6289598342b3\",\n \"name\": \"494c647c-02a4-4a85-8468-6289598342b3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:39:36.538874537Z\",\n \"endTime\": \"2026-09-16T13:08:44.880114684Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m35s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:98]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c10220\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-89n7j94drhpq\" to finish creating: GET https://rp.j5673216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/494c647c-02a4-4a85-8468-6289598342b3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/494c647c-02a4-4a85-8468-6289598342b3\",\n \"name\": \"494c647c-02a4-4a85-8468-6289598342b3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:39:36.538874537Z\",\n \"endTime\": \"2026-09-16T13:08:44.880114684Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m35s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fd2a0f4d0f5f1c60182768e15dbca249e721852c97b7142bc974301f30f229e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189557025673216","occurred_at":"2026-09-16T11:46:24Z","failed_at":"e2e","signature_id":"3ec1682204cd21163b91258f7fd79ed3ef2df2235e522890ff723136f72137a4","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d19280\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-z9lx64t7sbcs\" to finish creating: GET https://rp.j5673216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9a0788ae-1307-4f76-a260-f9ac34be043b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9a0788ae-1307-4f76-a260-f9ac34be043b\",\n \"name\": \"9a0788ae-1307-4f76-a260-f9ac34be043b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:39:59.621910893Z\",\n \"endTime\": \"2026-09-16T13:09:04.749574347Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 20m28s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d19280\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-z9lx64t7sbcs\" to finish creating: GET https://rp.j5673216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9a0788ae-1307-4f76-a260-f9ac34be043b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9a0788ae-1307-4f76-a260-f9ac34be043b\",\n \"name\": \"9a0788ae-1307-4f76-a260-f9ac34be043b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:39:59.621910893Z\",\n \"endTime\": \"2026-09-16T13:09:04.749574347Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 20m28s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","b414f524aec7c4ea3c6d464450b9af6f734740715a61aff1c3a8f53547f62641"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189503086923776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6917,"pr_state":"open","pr_sha":"55efd56d7b76c88f48195a55ad9df67aaf1f0f91","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T11:46:06Z","started_at":"2026-09-16T11:46:06Z","completed_at":"2026-09-16T13:51:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:39.826563473Z","timing_metadata_checked_at":"2026-09-18T15:40:39.826563473Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:39.829800685Z","region_metadata_checked_at":"2026-09-18T15:40:39.829800685Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"06bd0ce815ae087f47b780917c1eb7fc1aceef51083b6fb6582abaa77260e6da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189503086923776","occurred_at":"2026-09-16T11:46:06Z","failed_at":"e2e","signature_id":"c5a888e0079b8b0888371862a2ca292117eb1766fc5723aec9962ec739421dc6","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--79b927dmgkb5/cluster-hs-pre-b5vpk5 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101e8e0\u003e: \n failed to create HCP cluster cluster-hs-pre-b5vpk5: failed waiting for cluster=\"cluster-hs-pre-b5vpk5\" in resourcegroup=\"rg-hs-pre--79b927dmgkb5\" to finish creating: GET https://rp.j6923776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b7036fcd-648a-4938-a4d2-03f3c4c3a098\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b7036fcd-648a-4938-a4d2-03f3c4c3a098\",\n \"name\": \"b7036fcd-648a-4938-a4d2-03f3c4c3a098\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:41:50.885675594Z\",\n \"endTime\": \"2026-09-16T13:10:51.145718175Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m58s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--79b927dmgkb5/cluster-hs-pre-b5vpk5 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101e8e0\u003e: \n failed to create HCP cluster cluster-hs-pre-b5vpk5: failed waiting for cluster=\"cluster-hs-pre-b5vpk5\" in resourcegroup=\"rg-hs-pre--79b927dmgkb5\" to finish creating: GET https://rp.j6923776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b7036fcd-648a-4938-a4d2-03f3c4c3a098\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b7036fcd-648a-4938-a4d2-03f3c4c3a098\",\n \"name\": \"b7036fcd-648a-4938-a4d2-03f3c4c3a098\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:41:50.885675594Z\",\n \"endTime\": \"2026-09-16T13:10:51.145718175Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m58s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0f7d7a84fb6911be5de7e9def2875850630135bee2431325d507d124c7f9ceec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189503086923776","occurred_at":"2026-09-16T11:46:06Z","failed_at":"e2e","signature_id":"7fe3931d094682c8af6a0a3a715b07aac46854b46c2bbb9f6ad09b29f24ea4b6","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000706aa0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-dwmf5wnrnqjn\" to finish creating: GET https://rp.j6923776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7362da66-e5d5-454e-b456-4b2dce3364f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7362da66-e5d5-454e-b456-4b2dce3364f4\",\n \"name\": \"7362da66-e5d5-454e-b456-4b2dce3364f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:42:20.52077321Z\",\n \"endTime\": \"2026-09-16T13:11:21.200973872Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m2s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000706aa0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-dwmf5wnrnqjn\" to finish creating: GET https://rp.j6923776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7362da66-e5d5-454e-b456-4b2dce3364f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7362da66-e5d5-454e-b456-4b2dce3364f4\",\n \"name\": \"7362da66-e5d5-454e-b456-4b2dce3364f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:42:20.52077321Z\",\n \"endTime\": \"2026-09-16T13:11:21.200973872Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m2s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4ddcb924e97c1079fb470ab0f1882531dc89a65d3a6b682ff4999acd14f5ca3e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189503086923776","occurred_at":"2026-09-16T11:46:06Z","failed_at":"alert","signature_id":"382d6e1a50a58344a999c679c7ef69f1f3c041161e556f0be9d14b621d84b09e","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T12:57:26Z\n Ended: 2026-09-16T13:17:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6923776-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss41gibi7sp422ii8he0pq63nn52g37-private-kas\", pod=\"router-6b58c6bb5b-92m4m\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss41gibi7sp422ii8he0pq63nn52g37-private-kas/router-6b58c6bb5b-92m4m has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-16T12:57:26Z\n Ended: 2026-09-16T13:17:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6923776-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss41b4su9bid10hrqo3uiiuf5p3hgs4-b8r7o4e7s7g6i3a\", pod=\"router-b8d96bd58-thzdn\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss41b4su9bid10hrqo3uiiuf5p3hgs4-b8r7o4e7s7g6i3a/router-b8d96bd58-thzdn has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-16T12:57:26Z\n Ended: 2026-09-16T13:19:25Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6923776-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss416rp6lv3sb7n8d5pdpc3a57h0rqp-j4u9m8f9s6l0v4z\", pod=\"router-8fbb8c5f8-swtx6\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss416rp6lv3sb7n8d5pdpc3a57h0rqp-j4u9m8f9s6l0v4z/router-8fbb8c5f8-swtx6 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"8110c169691003c2dcbeaa3609460f2c3ba68176543d64e50d1fce0e08f16f25","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189503086923776","occurred_at":"2026-09-16T11:46:06Z","failed_at":"e2e","signature_id":"5b34bf27b7432e062ab59dae95c6e2c2ce5726c184df3b6d5751452cf7c1aac8","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000af3c40\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-hvkg6dzj54xx\" to finish creating: GET https://rp.j6923776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f44314ae-97e1-419e-bb21-2e04874f19c3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f44314ae-97e1-419e-bb21-2e04874f19c3\",\n \"name\": \"f44314ae-97e1-419e-bb21-2e04874f19c3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:42:40.715298605Z\",\n \"endTime\": \"2026-09-16T13:11:41.209044126Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m2s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000af3c40\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-hvkg6dzj54xx\" to finish creating: GET https://rp.j6923776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f44314ae-97e1-419e-bb21-2e04874f19c3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f44314ae-97e1-419e-bb21-2e04874f19c3\",\n \"name\": \"f44314ae-97e1-419e-bb21-2e04874f19c3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:42:40.715298605Z\",\n \"endTime\": \"2026-09-16T13:11:41.209044126Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m2s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d771e1465d9cc11b676b0942d536d4d8dd54d10981863ee8db4c88b4bb26eb58","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189503086923776","occurred_at":"2026-09-16T11:46:06Z","failed_at":"e2e","signature_id":"40e2d334636405f418ffb52c5061fbf30366656d327d9d8fc83b6e9d9afd9837","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-crjc9ggv5c5c/cluster-candidate-4-22-kthmtb should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001180160\u003e: \n failed to create HCP cluster cluster-candidate-4-22-kthmtb: failed waiting for cluster=\"cluster-candidate-4-22-kthmtb\" in resourcegroup=\"rg-candidate-4-22-crjc9ggv5c5c\" to finish creating: GET https://rp.j6923776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c590b0b-f53f-4d94-8c31-965c6434354b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c590b0b-f53f-4d94-8c31-965c6434354b\",\n \"name\": \"9c590b0b-f53f-4d94-8c31-965c6434354b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:42:02.207518072Z\",\n \"endTime\": \"2026-09-16T13:11:11.226520946Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 21m34s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-crjc9ggv5c5c/cluster-candidate-4-22-kthmtb should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001180160\u003e: \n failed to create HCP cluster cluster-candidate-4-22-kthmtb: failed waiting for cluster=\"cluster-candidate-4-22-kthmtb\" in resourcegroup=\"rg-candidate-4-22-crjc9ggv5c5c\" to finish creating: GET https://rp.j6923776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c590b0b-f53f-4d94-8c31-965c6434354b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c590b0b-f53f-4d94-8c31-965c6434354b\",\n \"name\": \"9c590b0b-f53f-4d94-8c31-965c6434354b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:42:02.207518072Z\",\n \"endTime\": \"2026-09-16T13:11:11.226520946Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 21m34s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100188464501755904","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"0ed6e687bde31cad45ae95be2a5517dad170d6f7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T11:42:04Z","started_at":"2026-09-16T11:42:04Z","completed_at":"2026-09-16T13:49:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:05.239301526Z","timing_metadata_checked_at":"2026-09-18T15:40:05.239301526Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:05.242335837Z","region_metadata_checked_at":"2026-09-18T15:40:05.242335837Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"5f271fb1ebc7b8aabd0f31cfe6e26b7b819a80ec1ef86d53008f5d68b1ec0b93","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100188464501755904","occurred_at":"2026-09-16T11:42:04Z","failed_at":"e2e","signature_id":"139ea4eb9a07c176246f76b251911c3691a6cbbed5076e752b52b4042c281baf","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc001a980c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-gztjzzfvghvl\" to finish deleting: GET https://rp.j1755904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/067d2d4c-07a0-4859-8081-90eb78e5b132\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/067d2d4c-07a0-4859-8081-90eb78e5b132\",\n \"name\": \"067d2d4c-07a0-4859-8081-90eb78e5b132\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:52:08.640021777Z\",\n \"endTime\": \"2026-09-16T13:16:14.927620259Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss3v2g1581u9fmlnc5tsgv7vcs4icad still exists (deletion dispatched at 2026-09-16T12:52:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc001a980c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-gztjzzfvghvl\" to finish deleting: GET https://rp.j1755904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/067d2d4c-07a0-4859-8081-90eb78e5b132\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/067d2d4c-07a0-4859-8081-90eb78e5b132\",\n \"name\": \"067d2d4c-07a0-4859-8081-90eb78e5b132\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:52:08.640021777Z\",\n \"endTime\": \"2026-09-16T13:16:14.927620259Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss3v2g1581u9fmlnc5tsgv7vcs4icad still exists (deletion dispatched at 2026-09-16T12:52:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6b970fbc194a31318296f1c27ec92ae423c279277b46b46c5a696f0b2ed7daf6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100188464501755904","occurred_at":"2026-09-16T11:42:04Z","failed_at":"e2e","signature_id":"d4cf5fc8868b958b193d13f740ed514c1bae0dd9b8489c2b78271801c3f259d7","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc0013980d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-9gp2cv4556mc\" to finish deleting: GET https://rp.j1755904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f8e89b79-b819-4d57-beb6-45d5fd38c76b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f8e89b79-b819-4d57-beb6-45d5fd38c76b\",\n \"name\": \"f8e89b79-b819-4d57-beb6-45d5fd38c76b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:52:15.549173368Z\",\n \"endTime\": \"2026-09-16T13:16:24.936632758Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss3unt2thcjv9iur8dobv2ob7ghr15h still exists (deletion dispatched at 2026-09-16T12:52:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc0013980d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-9gp2cv4556mc\" to finish deleting: GET https://rp.j1755904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f8e89b79-b819-4d57-beb6-45d5fd38c76b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f8e89b79-b819-4d57-beb6-45d5fd38c76b\",\n \"name\": \"f8e89b79-b819-4d57-beb6-45d5fd38c76b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:52:15.549173368Z\",\n \"endTime\": \"2026-09-16T13:16:24.936632758Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss3unt2thcjv9iur8dobv2ob7ghr15h still exists (deletion dispatched at 2026-09-16T12:52:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8f6238051d8c819dd5d27851843d008c46fc418bb91c7385cb38e53049bf6849","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100188464501755904","occurred_at":"2026-09-16T11:42:04Z","failed_at":"e2e","signature_id":"85767451e4aa0503e59ddd278738b04f06b742245943e9225dbc7e3560e04b05","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc0017780a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-225ts6vlpjr6\" to finish deleting: GET https://rp.j1755904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2359d86d-b689-4fd0-915b-2d216b861e83\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2359d86d-b689-4fd0-915b-2d216b861e83\",\n \"name\": \"2359d86d-b689-4fd0-915b-2d216b861e83\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:52:03.476346261Z\",\n \"endTime\": \"2026-09-16T13:16:04.956406239Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss3uucne5o52iff154le845qrjeepnu still exists (deletion dispatched at 2026-09-16T12:52:04Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:289]: Unexpected error:\n \u003c*errors.joinError | 0xc0017780a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-225ts6vlpjr6\" to finish deleting: GET https://rp.j1755904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2359d86d-b689-4fd0-915b-2d216b861e83\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2359d86d-b689-4fd0-915b-2d216b861e83\",\n \"name\": \"2359d86d-b689-4fd0-915b-2d216b861e83\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:52:03.476346261Z\",\n \"endTime\": \"2026-09-16T13:16:04.956406239Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss3uucne5o52iff154le845qrjeepnu still exists (deletion dispatched at 2026-09-16T12:52:04Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a106963ec6bbb94a873b8c3c45c07b69783d4ee577d4ca2e9e2b445b8c7032b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100188464501755904","occurred_at":"2026-09-16T11:42:04Z","failed_at":"e2e","signature_id":"b62e9e3d53137e1d7201b9bf6b9b273592900eb1294c925f09b1fe4ee1b51c90","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:480]: Timed out after 34.093s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000ff3040\u003e: \n permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /endpoints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /events: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /persistentvolumeclaims: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for get /persistentvolumes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /serviceaccounts: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /resourcequotas: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /limitranges: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /replicationcontrollers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for get /componentstatuses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for get /pods/log: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apps/deployments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apps/daemonsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apps/statefulsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apps/replicasets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list batch/jobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list batch/cronjobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list networking.k8s.io/networkpolicies: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list networking.k8s.io/ingresses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list rbac.authorization.k8s.io/clusterroles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list rbac.authorization.k8s.io/clusterrolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list rbac.authorization.k8s.io/roles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list rbac.authorization.k8s.io/rolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list storage.k8s.io/storageclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list storage.k8s.io/volumeattachments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apiextensions.k8s.io/customresourcedefinitions: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list policy/poddisruptionbudgets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list autoscaling/horizontalpodautoscalers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list coordination.k8s.io/leases: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list discovery.k8s.io/endpointslices: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list certificates.k8s.io/certificatesigningrequests: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list scheduling.k8s.io/priorityclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list node.k8s.io/runtimeclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list admissionregistration.k8s.io/mutatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list admissionregistration.k8s.io/validatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:480]: Timed out after 34.093s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000ff3040\u003e: \n permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /endpoints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /events: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /persistentvolumeclaims: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for get /persistentvolumes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /serviceaccounts: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /resourcequotas: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /limitranges: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /replicationcontrollers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for get /componentstatuses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for get /pods/log: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apps/deployments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apps/daemonsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apps/statefulsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apps/replicasets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list batch/jobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list batch/cronjobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list networking.k8s.io/networkpolicies: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list networking.k8s.io/ingresses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list rbac.authorization.k8s.io/clusterroles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list rbac.authorization.k8s.io/clusterrolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list rbac.authorization.k8s.io/roles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list rbac.authorization.k8s.io/rolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list storage.k8s.io/storageclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list storage.k8s.io/volumeattachments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apiextensions.k8s.io/customresourcedefinitions: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list policy/poddisruptionbudgets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list autoscaling/horizontalpodautoscalers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list coordination.k8s.io/leases: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list discovery.k8s.io/endpointslices: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list certificates.k8s.io/certificatesigningrequests: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list scheduling.k8s.io/priorityclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list node.k8s.io/runtimeclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list admissionregistration.k8s.io/mutatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list admissionregistration.k8s.io/validatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"50012cc84060fc59db5e840f9933bf1a542baf6598061d9d1e54da53aa955055","failure_pattern":"permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews","search_query":"permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["50012cc84060fc59db5e840f9933bf1a542baf6598061d9d1e54da53aa955055","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6726/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100185438990897152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6726,"pr_state":"closed","pr_sha":"92641d0bf445357c6af705106ec4bd7ff3f095c2","final_merged_sha":"84e4e522be467834675e1c0d8dc52b471e0d01ae","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T11:30:02Z","started_at":"2026-09-16T11:30:02Z","completed_at":"2026-09-16T13:26:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:19.680098497Z","timing_metadata_checked_at":"2026-09-18T15:40:19.680098497Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:19.683633909Z","region_metadata_checked_at":"2026-09-18T15:40:19.683633909Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"de90d6f39ac05e8c22d57af6894472d6ad8ef26342a2d8355950e1607f4b9e7c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6726/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100185438990897152","occurred_at":"2026-09-16T11:30:02Z","failed_at":"alert","signature_id":"e361abead0603e0446eac1b70560a55fbc49aa8e7044098f3920029a67178a2a","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T12:49:06Z\n Ended: 2026-09-16T13:01:06Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0897152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.188:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0897152/providers/microsoft.monitor/accounts/hcps-j0897152\", namespace=\"ocm-arohcpci01-2ss3lg4qnv008n1sgqgucdnh6ucncan0-j1s9x0r6b8u1v3h\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-8rm4f\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2ss3lg4qnv008n1sgqgucdnh6ucncan0-j1s9x0r6b8u1v3h/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-16T12:49:07Z\n Ended: 2026-09-16T12:56:03Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0897152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.188:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0897152/providers/microsoft.monitor/accounts/hcps-j0897152\", namespace=\"ocm-arohcpci01-2ss3lg4qnv008n1sgqgucdnh6ucncan0-j1s9x0r6b8u1v3h\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-8rm4f\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2ss3lg4qnv008n1sgqgucdnh6ucncan0-j1s9x0r6b8u1v3h/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"f59d268345f8e48fe7b1b3324d0bd3715e0d3cbbfc467dc96879938ce45390e5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6726/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100185438990897152","occurred_at":"2026-09-16T11:30:02Z","failed_at":"alert","signature_id":"5d5dbfb417b6897cd24a2f1e6ad614604212fa121b9f81e032502d4c16e94ed3","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 6 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T12:47:51Z\n Ended: 2026-09-16T13:06:54Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0897152-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-lswwb\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-lswwb (namespace arobit) on cluster ci01-j0897152-mgmt-2 has averaged 174.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-16T12:47:51Z\n Ended: 2026-09-16T13:04:57Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0897152-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-664vn\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-664vn (namespace arobit) on cluster ci01-j0897152-svc has averaged 169.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-16T12:47:51Z\n Ended: 2026-09-16T13:10:52Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0897152-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-84794bbc8-5fmm4\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-84794bbc8-5fmm4 (namespace clusters-service) on cluster ci01-j0897152-svc has averaged 218.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-16T12:47:51Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0897152-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-655696f65-mr299\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-655696f65-mr299 (namespace aro-hcp) on cluster ci01-j0897152-svc has averaged 400.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-16T12:47:51Z\n Ended: 2026-09-16T12:55:52Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0897152-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-84794bbc8-wt8kz\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-84794bbc8-wt8kz (namespace clusters-service) on cluster ci01-j0897152-svc has averaged 759.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Resolved\n Started: 2026-09-16T12:54:52Z\n Ended: 2026-09-16T13:05:51Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0897152-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-cbmhn\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-cbmhn (namespace arobit) on cluster ci01-j0897152-mgmt-1 has averaged 131.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"fa0b39c24f0aae0dca061113771f6226d92a56a317b83d092656d7ad43367167","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6726/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100185438990897152","occurred_at":"2026-09-16T11:30:02Z","failed_at":"e2e","signature_id":"681376edcfb91afa0eef4e45ea02cce5668cf8a6c8cbfde456dd6c90a347a910","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001092780\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-jqqcvh44krzk\" to finish creating: GET https://rp.j0897152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ad8bf203-db0b-47d7-bd3b-a158a50c55a1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ad8bf203-db0b-47d7-bd3b-a158a50c55a1\",\n \"name\": \"ad8bf203-db0b-47d7-bd3b-a158a50c55a1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:17:03.393436374Z\",\n \"endTime\": \"2026-09-16T12:46:10.759018774Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001092780\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-jqqcvh44krzk\" to finish creating: GET https://rp.j0897152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ad8bf203-db0b-47d7-bd3b-a158a50c55a1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ad8bf203-db0b-47d7-bd3b-a158a50c55a1\",\n \"name\": \"ad8bf203-db0b-47d7-bd3b-a158a50c55a1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:17:03.393436374Z\",\n \"endTime\": \"2026-09-16T12:46:10.759018774Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100177539514241024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6952,"pr_state":"closed","pr_sha":"a4de05dadb773673a7d1a63ee2677dbb182bda10","final_merged_sha":"a4de05dadb773673a7d1a63ee2677dbb182bda10","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T10:58:39Z","started_at":"2026-09-16T10:58:39Z","completed_at":"2026-09-16T13:14:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:57.072582014Z","timing_metadata_checked_at":"2026-09-18T15:40:57.072582014Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:57.075723125Z","region_metadata_checked_at":"2026-09-18T15:40:57.075723125Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"ac1cc65518d411744d27cac3d10a286649a075a4e000b08c869fc144574826b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100177539514241024","occurred_at":"2026-09-16T10:58:39Z","failed_at":"e2e","signature_id":"4035d63aa7b2b3c2922026477a9808247a9246b7c7d1f67b92c709fe54aaae23","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001ff26c0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-g5s8ftp9hh9t\" to finish creating: GET https://rp.j4241024.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06ef0feb-5e7e-4993-bd90-39657ca016bf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06ef0feb-5e7e-4993-bd90-39657ca016bf\",\n \"name\": \"06ef0feb-5e7e-4993-bd90-39657ca016bf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:18:53.259035342Z\",\n \"endTime\": \"2026-09-16T12:37:53.675934016Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001ff26c0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-g5s8ftp9hh9t\" to finish creating: GET https://rp.j4241024.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06ef0feb-5e7e-4993-bd90-39657ca016bf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06ef0feb-5e7e-4993-bd90-39657ca016bf\",\n \"name\": \"06ef0feb-5e7e-4993-bd90-39657ca016bf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T12:18:53.259035342Z\",\n \"endTime\": \"2026-09-16T12:37:53.675934016Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6779/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100177208487186432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6779,"pr_state":"open","pr_sha":"cd2fcb539c5a6b610c348d13df4e23f77e3e5130","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T10:57:20Z","started_at":"2026-09-16T10:57:20Z","completed_at":"2026-09-16T11:24:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:21.187002018Z","timing_metadata_checked_at":"2026-09-18T15:40:21.187002018Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:21.190093029Z","region_metadata_checked_at":"2026-09-19T03:30:34.428038461Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"2a584557c70df2688ccc991371a53eabf13e6bbe77502b895eb63e694019dea1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6779/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100177208487186432","occurred_at":"2026-09-16T10:57:20Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100176649516486656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6952,"pr_state":"closed","pr_sha":"a4de05dadb773673a7d1a63ee2677dbb182bda10","final_merged_sha":"a4de05dadb773673a7d1a63ee2677dbb182bda10","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T10:54:59Z","started_at":"2026-09-16T10:54:59Z","completed_at":"2026-09-16T13:12:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:22.927992142Z","timing_metadata_checked_at":"2026-09-18T15:41:22.927992142Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:22.931102053Z","region_metadata_checked_at":"2026-09-18T15:41:22.931102053Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"b21bf85c5dc86658a31699ac265c0b8391f700e35f4119b187f54e24d3f41009","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100176649516486656","occurred_at":"2026-09-16T10:54:59Z","failed_at":"alert","signature_id":"f201db6f99c06ca51a3a0a3934aae0846a6ea6ba393747b6f6edc6872fb99a72","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 15 time(s) (4 unknown, 11 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T12:06:16Z\n Ended: 2026-09-16T12:16:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6486656-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss3bi7r1h4v48sj30qba1fo8pvrqm7f-l6r4d7v5o3c4z2a\", pod=\"kube-apiserver-854cf58859-z8glp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss3bi7r1h4v48sj30qba1fo8pvrqm7f-l6r4d7v5o3c4z2a/kube-apiserver-854cf58859-z8glp has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-16T12:07:18Z\n Ended: 2026-09-16T12:17:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6486656-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss3b4g6moo56neqpa7uf1if0ci5u32q-k5p9e7w4i8s6c1y\", pod=\"kube-apiserver-76cdc4f787-whddh\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss3b4g6moo56neqpa7uf1if0ci5u32q-k5p9e7w4i8s6c1y/kube-apiserver-76cdc4f787-whddh has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-16T12:07:18Z\n Ended: 2026-09-16T12:17:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6486656-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss3b570knsufmrkkurvp62eg2ftj2qm-e6i6j3u6a5v6u0v\", pod=\"etcd-0\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss3b570knsufmrkkurvp62eg2ftj2qm-e6i6j3u6a5v6u0v/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-16T12:07:18Z\n Ended: 2026-09-16T12:16:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6486656-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss3b570knsufmrkkurvp62eg2ftj2qm-e6i6j3u6a5v6u0v\", pod=\"ignition-server-79c7574ccc-qw8dp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss3b570knsufmrkkurvp62eg2ftj2qm-e6i6j3u6a5v6u0v/ignition-server-79c7574ccc-qw8dp has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100170313131102208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6886,"pr_state":"open","pr_sha":"a6aa6a03a137ec55e3f4f5d06cc620e6bf6dd7f3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T10:29:56Z","started_at":"2026-09-16T10:29:56Z","completed_at":"2026-09-16T12:33:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:31.487596574Z","timing_metadata_checked_at":"2026-09-18T15:40:31.487596574Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:31.490916186Z","region_metadata_checked_at":"2026-09-18T15:40:31.490916186Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"973dd8e23bb4d9965ecb98f92bc6fb46090976d4ac5e24a3ac1254397e38290e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100170313131102208","occurred_at":"2026-09-16T10:29:56Z","failed_at":"e2e","signature_id":"f02c430f8d6e71033486f073da10a12fce79eb5455b645129c51f20c2adc1b42","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00019f020\u003e: \n failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-t55rrq\" in resourcegroup=\"rg-np-skip-minor-t55rrq-wdl65v69n5wm\" to finish creating: GET https://rp.j1102208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a0da7c7c-5ccf-448f-9d31-2330d22070c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a0da7c7c-5ccf-448f-9d31-2330d22070c5\",\n \"name\": \"a0da7c7c-5ccf-448f-9d31-2330d22070c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T11:32:30.704689728Z\",\n \"endTime\": \"2026-09-16T11:51:34.799712169Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00019f020\u003e: \n failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-t55rrq\" in resourcegroup=\"rg-np-skip-minor-t55rrq-wdl65v69n5wm\" to finish creating: GET https://rp.j1102208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a0da7c7c-5ccf-448f-9d31-2330d22070c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a0da7c7c-5ccf-448f-9d31-2330d22070c5\",\n \"name\": \"a0da7c7c-5ccf-448f-9d31-2330d22070c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T11:32:30.704689728Z\",\n \"endTime\": \"2026-09-16T11:51:34.799712169Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6694/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100169554075324416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6694,"pr_state":"closed","pr_sha":"f1437e7522fb6efcd77f296c45896ba121660db7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T10:26:55Z","started_at":"2026-09-16T10:26:55Z","completed_at":"2026-09-16T11:26:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:13.598726322Z","timing_metadata_checked_at":"2026-09-18T15:40:13.598726322Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:13.603022138Z","region_metadata_checked_at":"2026-09-18T15:40:13.603022138Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"27a491793712efeb73fc7c00b3b530d3ce99246cf294468864ef3cd4ef668d86","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6694/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100169554075324416","occurred_at":"2026-09-16T10:26:55Z","failed_at":"provision","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","test_suite":"step graph","failure_text":"time=2026-09-16T10:50:03.391Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-16T10:50:04.001Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-16T10:50:05.394Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=39c0bfe6f31a9b99040636b0dd2da01bd62593c0723e012a25f7c174f8b2444b portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5324416-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F39c0bfe6f31a9b99040636b0dd2da01bd62593c0723e012a25f7c174f8b2444b\ntime=2026-09-16T11:20:03.104Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100164648379617280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6898,"pr_state":"closed","pr_sha":"e648df2a06ccc56def0ce2544593a2720ec3ff72","final_merged_sha":"3f0521ff633af0368f049bfead3093e31565b50f","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-16T10:07:25Z","started_at":"2026-09-16T10:07:25Z","completed_at":"2026-09-16T12:07:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:35.180543555Z","timing_metadata_checked_at":"2026-09-18T15:40:35.180543555Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:35.184312569Z","region_metadata_checked_at":"2026-09-18T15:40:35.184312569Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100164075236364288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6688,"pr_state":"open","pr_sha":"e621848578e865601181f8eeed88165481777533","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T10:05:09Z","started_at":"2026-09-16T10:05:09Z","completed_at":"2026-09-16T11:57:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:11.66417746Z","timing_metadata_checked_at":"2026-09-18T15:40:11.66417746Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:11.667276871Z","region_metadata_checked_at":"2026-09-18T15:40:11.667276871Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"4ede72740be1658f29b0357811f40374652c9828953c5a8b79b8b48e3044686a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100164075236364288","occurred_at":"2026-09-16T10:05:09Z","failed_at":"e2e","signature_id":"05b8123730c99aae90bf4a2eb92268ce4dedb8de1e1191eb555f549c617a26dd","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:111]: failed to create node pool \"np-w5pk5g\" for cluster \"cluster-hs-pre-w5pk5g\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00108f320\u003e: \n failed to create NodePool np-w5pk5g: failed waiting for nodepool=\"np-w5pk5g\" for cluster \"cluster-hs-pre-w5pk5g\" in resourcegroup=\"rg-hs-pre--tkbl565tmmql\" to finish creating: GET https://rp.j6364288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2191135-7ef2-4813-a443-f5ccab881558\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2191135-7ef2-4813-a443-f5ccab881558\",\n \"name\": \"f2191135-7ef2-4813-a443-f5ccab881558\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T11:03:25.0216776Z\",\n \"endTime\": \"2026-09-16T11:22:33.00315027Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:111]: failed to create node pool \"np-w5pk5g\" for cluster \"cluster-hs-pre-w5pk5g\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00108f320\u003e: \n failed to create NodePool np-w5pk5g: failed waiting for nodepool=\"np-w5pk5g\" for cluster \"cluster-hs-pre-w5pk5g\" in resourcegroup=\"rg-hs-pre--tkbl565tmmql\" to finish creating: GET https://rp.j6364288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2191135-7ef2-4813-a443-f5ccab881558\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2191135-7ef2-4813-a443-f5ccab881558\",\n \"name\": \"f2191135-7ef2-4813-a443-f5ccab881558\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T11:03:25.0216776Z\",\n \"endTime\": \"2026-09-16T11:22:33.00315027Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7ac81eb46321ec8e3d7f26f3bc99194beaf1bd8dfe8c12d0f92de71f6b565082","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100164075236364288","occurred_at":"2026-09-16T10:05:09Z","failed_at":"e2e","signature_id":"186ac175cc37e33f1eaa9e81000b9ad2fcf1670566cc6849f36ba3c782d224b2","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.000s.\nat least one schedule should have a LastBackupTime\nExpected\n \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.000s.\nat least one schedule should have a LastBackupTime\nExpected\n \u003cbool\u003e: ...\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}},{"row_id":"cb3e248111230a43445bfdbceeecbb96a56a98cdcf38fcc661f3958cef087568","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100164075236364288","occurred_at":"2026-09-16T10:05:09Z","failed_at":"e2e","signature_id":"59512808dbcb20b06030c9fde07924554878fee722f13b2a06c73ba1d94bd2f9","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:204]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc00230a300\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ql5j7w6fx6vn\" to finish creating: GET https://rp.j6364288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5fde05ba-68c2-437a-84ea-82765b356728\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5fde05ba-68c2-437a-84ea-82765b356728\",\n \"name\": \"5fde05ba-68c2-437a-84ea-82765b356728\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T11:04:09.351110934Z\",\n \"endTime\": \"2026-09-16T11:04:15.302518412Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:204]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc00230a300\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ql5j7w6fx6vn\" to finish creating: GET https://rp.j6364288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5fde05ba-68c2-437a-84ea-82765b356728\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5fde05ba-68c2-437a-84ea-82765b356728\",\n \"name\": \"5fde05ba-68c2-437a-84ea-82765b356728\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T11:04:09.351110934Z\",\n \"endTime\": \"2026-09-16T11:04:15.302518412Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fe24c8b4b28d3d2b1937803a92704c44ec574f3d1c3b740ec6d97282a1e18d39","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","fe24c8b4b28d3d2b1937803a92704c44ec574f3d1c3b740ec6d97282a1e18d39"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6791,"pr_state":"closed","pr_sha":"764ee60fa32d166f92c6c286f96007787ee189de","final_merged_sha":"00c5d478babb26a92f08e45ee9f4c322bafd7507","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T09:56:36Z","started_at":"2026-09-16T09:56:36Z","completed_at":"2026-09-16T12:14:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:22.744785622Z","timing_metadata_checked_at":"2026-09-18T15:40:22.744785622Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:22.748361535Z","region_metadata_checked_at":"2026-09-18T15:40:22.748361535Z"},"failed_at":["e2e"],"failed_test_count":8,"occurrences":[{"row_id":"0109eb57fba020aaa069d2c5ccf3265ef5a67716282423b5acb85798e0e32230","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","failed_at":"e2e","signature_id":"91a400cbca3134720c48834459e3002d3cce5a22f2d380c716d75f65f9d75177","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--9b4s75wtf446/cluster-hs-pre-vpfc9j should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001505c0\u003e: \n failed to create HCP cluster cluster-hs-pre-vpfc9j: failed waiting for cluster=\"cluster-hs-pre-vpfc9j\" in resourcegroup=\"rg-hs-pre--9b4s75wtf446\" to finish creating: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/be3e1cc0-6007-4f70-bd41-a633b0402774\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/be3e1cc0-6007-4f70-bd41-a633b0402774\",\n \"name\": \"be3e1cc0-6007-4f70-bd41-a633b0402774\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T10:52:03.039188877Z\",\n \"endTime\": \"2026-09-16T11:21:09.718773632Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--9b4s75wtf446/cluster-hs-pre-vpfc9j should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001505c0\u003e: \n failed to create HCP cluster cluster-hs-pre-vpfc9j: failed waiting for cluster=\"cluster-hs-pre-vpfc9j\" in resourcegroup=\"rg-hs-pre--9b4s75wtf446\" to finish creating: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/be3e1cc0-6007-4f70-bd41-a633b0402774\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/be3e1cc0-6007-4f70-bd41-a633b0402774\",\n \"name\": \"be3e1cc0-6007-4f70-bd41-a633b0402774\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T10:52:03.039188877Z\",\n \"endTime\": \"2026-09-16T11:21:09.718773632Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"081e4b881283ebea582403be69fae264fb2e9ac76aed47b94474869d69f5ff90","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","failed_at":"e2e","signature_id":"b85f0369f91fe32b9c9c2062c2413bb6bf553dd1bd69f6e51a7a27c7ba432aee","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000db6000\u003e: \n failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-xn5sprrvmhpp\" to finish creating: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7e44efb7-2fb2-4dd3-851e-724f837768e3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7e44efb7-2fb2-4dd3-851e-724f837768e3\",\n \"name\": \"7e44efb7-2fb2-4dd3-851e-724f837768e3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T11:10:24.772921169Z\",\n \"endTime\": \"2026-09-16T11:29:29.692653081Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000db6000\u003e: \n failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-xn5sprrvmhpp\" to finish creating: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7e44efb7-2fb2-4dd3-851e-724f837768e3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7e44efb7-2fb2-4dd3-851e-724f837768e3\",\n \"name\": \"7e44efb7-2fb2-4dd3-851e-724f837768e3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T11:10:24.772921169Z\",\n \"endTime\": \"2026-09-16T11:29:29.692653081Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0b5bfefbdb765bd9f19b6428091cfb7581aa95691de3a86a0d79b16618874a81","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","failed_at":"e2e","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"52536575bd23049427997d4c5915013048f9751332857bb0243da2c967fe2521","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","failed_at":"e2e","signature_id":"874e6ea76f2ba4acc89e73ac761bf0c81c038b75f11db73023de66c025986ab7","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00108c078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-mzfrqt8tf9wq\" to finish deleting: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9f01e6a-fcf9-4bbb-aeeb-61ee6e0ac502\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9f01e6a-fcf9-4bbb-aeeb-61ee6e0ac502\",\n \"name\": \"e9f01e6a-fcf9-4bbb-aeeb-61ee6e0ac502\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T11:18:42.744808437Z\",\n \"endTime\": \"2026-09-16T11:42:49.617012035Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00108c078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-mzfrqt8tf9wq\" to finish deleting: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9f01e6a-fcf9-4bbb-aeeb-61ee6e0ac502\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9f01e6a-fcf9-4bbb-aeeb-61ee6e0ac502\",\n \"name\": \"e9f01e6a-fcf9-4bbb-aeeb-61ee6e0ac502\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T11:18:42.744808437Z\",\n \"endTime\": \"2026-09-16T11:42:49.617012035Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3bc88c6c8876f1c2ea5668dd924adc7c81d6e0349b9850342e67442ed3fe914","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8bedb0d47a6c2289a3b7f7d773a379b945ba6b5b9909f81a8e9a8ed589d1505b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","failed_at":"e2e","signature_id":"84f288aa90f1963969fa6df3b97105ac35a5dcb0b0f4f283ef7675cb82451eeb","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011044c0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-95l622wn4sm9\" to finish creating: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b24629a-f978-4a3d-9d8d-8d5b6776590f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b24629a-f978-4a3d-9d8d-8d5b6776590f\",\n \"name\": \"1b24629a-f978-4a3d-9d8d-8d5b6776590f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T10:52:02.869364535Z\",\n \"endTime\": \"2026-09-16T11:21:09.719035401Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011044c0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-95l622wn4sm9\" to finish creating: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b24629a-f978-4a3d-9d8d-8d5b6776590f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b24629a-f978-4a3d-9d8d-8d5b6776590f\",\n \"name\": \"1b24629a-f978-4a3d-9d8d-8d5b6776590f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T10:52:02.869364535Z\",\n \"endTime\": \"2026-09-16T11:21:09.719035401Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1d42c7dad9427d57e1b19754ffe767a1fc187725c1c8da2f03ad1f6733ce9b5d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9f75cbeed5d8d1d53d0bb322b1f6bff52bedbe41ca53aea22af24c16c3617e5e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","failed_at":"e2e","signature_id":"55ad0310438d8f56a4a299f6bab3b8eead74cd3a5f7c47eb326a8464bccf50a4","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-vm24lv8gf4qb/cluster-candidate-4-21-llvd77 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011b0380\u003e: \n failed to create HCP cluster cluster-candidate-4-21-llvd77: failed waiting for cluster=\"cluster-candidate-4-21-llvd77\" in resourcegroup=\"rg-candidate-4-21-vm24lv8gf4qb\" to finish creating: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a4429abc-10d8-4151-ad1a-6f415aa5475f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a4429abc-10d8-4151-ad1a-6f415aa5475f\",\n \"name\": \"a4429abc-10d8-4151-ad1a-6f415aa5475f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T10:52:02.803955113Z\",\n \"endTime\": \"2026-09-16T11:21:09.695382854Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-vm24lv8gf4qb/cluster-candidate-4-21-llvd77 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011b0380\u003e: \n failed to create HCP cluster cluster-candidate-4-21-llvd77: failed waiting for cluster=\"cluster-candidate-4-21-llvd77\" in resourcegroup=\"rg-candidate-4-21-vm24lv8gf4qb\" to finish creating: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a4429abc-10d8-4151-ad1a-6f415aa5475f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a4429abc-10d8-4151-ad1a-6f415aa5475f\",\n \"name\": \"a4429abc-10d8-4151-ad1a-6f415aa5475f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T10:52:02.803955113Z\",\n \"endTime\": \"2026-09-16T11:21:09.695382854Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"aa09f86a0efea7c699e1b96e457fbef15dcde29e1c1213662bdd5e8f1cb3d5e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","failed_at":"e2e","signature_id":"7299032a0750fba48148c45abb91daa5c3ae005efe59b617e7dc255e091f7a66","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aaaa60\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6mp8vkxcbh5s\" to finish creating: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d35dfb81-23c2-4a18-80a3-7d5bf5893728\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d35dfb81-23c2-4a18-80a3-7d5bf5893728\",\n \"name\": \"d35dfb81-23c2-4a18-80a3-7d5bf5893728\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T10:51:43.295429479Z\",\n \"endTime\": \"2026-09-16T11:20:49.710317459Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aaaa60\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6mp8vkxcbh5s\" to finish creating: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d35dfb81-23c2-4a18-80a3-7d5bf5893728\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d35dfb81-23c2-4a18-80a3-7d5bf5893728\",\n \"name\": \"d35dfb81-23c2-4a18-80a3-7d5bf5893728\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T10:51:43.295429479Z\",\n \"endTime\": \"2026-09-16T11:20:49.710317459Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1d42c7dad9427d57e1b19754ffe767a1fc187725c1c8da2f03ad1f6733ce9b5d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c9655e36e3917e379346cf2a24af9c635eb0af59e429239b4347f5a9fe73ffa3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","failed_at":"e2e","signature_id":"b82de203b62ff6756c30d031383d9f2c1cda2b52933916b1090a4b903f7e89b9","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:98]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001082360\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-mqnmkb84gsqq\" to finish creating: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a86c332b-3661-447d-86c0-3d610ff0d6c0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a86c332b-3661-447d-86c0-3d610ff0d6c0\",\n \"name\": \"a86c332b-3661-447d-86c0-3d610ff0d6c0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T10:51:52.760340073Z\",\n \"endTime\": \"2026-09-16T11:20:59.608470437Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:98]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001082360\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-mqnmkb84gsqq\" to finish creating: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a86c332b-3661-447d-86c0-3d610ff0d6c0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a86c332b-3661-447d-86c0-3d610ff0d6c0\",\n \"name\": \"a86c332b-3661-447d-86c0-3d610ff0d6c0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T10:51:52.760340073Z\",\n \"endTime\": \"2026-09-16T11:20:59.608470437Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["1d42c7dad9427d57e1b19754ffe767a1fc187725c1c8da2f03ad1f6733ce9b5d","9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","f3bc88c6c8876f1c2ea5668dd924adc7c81d6e0349b9850342e67442ed3fe914","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161854826352640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6820,"pr_state":"open","pr_sha":"c99d95a578f89e3e4c3bfc476a031bb4b59056f4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T09:56:19Z","started_at":"2026-09-16T09:56:19Z","completed_at":"2026-09-16T11:11:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:25.307571443Z","timing_metadata_checked_at":"2026-09-18T15:40:25.307571443Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:25.311022455Z","region_metadata_checked_at":"2026-09-18T15:40:25.311022455Z"},"failed_at":["alert","provision"],"failed_test_count":4,"occurrences":[{"row_id":"7165e7804de91c0684c1b3f428ce9fba8aa8fb735bcdc9b20d4f74f944c6440f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161854826352640","occurred_at":"2026-09-16T09:56:19Z","failed_at":"alert","signature_id":"f46fa77bbf961b7174b5b28e58ea0b344fb9d8b591b31027fca64620b0c955e9","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T10:41:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6352640-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-4526h\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-4526h has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-16T10:41:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6352640-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-q59vx\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-q59vx has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-16T10:41:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6352640-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-h4n49\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-h4n49 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-16T10:41:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6352640-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-6rpq8\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-6rpq8 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"a42984c02e9b03de3d9a8e26a403cf4da2f881f0282be26fad94b33113ae6487","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161854826352640","occurred_at":"2026-09-16T09:56:19Z","failed_at":"alert","signature_id":"40d8f345b61cc5a9851d9600ef3889f30540178c4979cc679d48f241649fe011","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T10:51:08Z\n Severity: Sev3\n Labels: alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j6352640-svc\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"warning\"\n Description: The Arobit forwarder metrics endpoint on cluster ci01-j6352640-svc has been unreachable for 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired"}},{"row_id":"d2ea9f2f80b80abd9cffec01e3a166a4e2e2395d573554d44e53529a76110ca0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161854826352640","occurred_at":"2026-09-16T09:56:19Z","failed_at":"alert","signature_id":"73664eccd132c9d84475a61b44857f75314a6635b79c98d2ae19c6ecf756089a","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T10:51:25Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j6352640-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.74:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6352640/providers/microsoft.monitor/accounts/services-j6352640\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5979b8454d-4mlt5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-16T10:51:25Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j6352640-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.74:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6352640/providers/microsoft.monitor/accounts/services-j6352640\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5979b8454d-4mlt5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired"}},{"row_id":"ff930e2a12d7312b84508deace2f29b3c6a44712bf361e2cc782eab2c5628abd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161854826352640","occurred_at":"2026-09-16T09:56:19Z","failed_at":"provision","signature_id":"64a318e95cdcfb12234888d476e960be3b284497df057c04c923477d134c5a2a","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","test_suite":"step graph","failure_text":"time=2026-09-16T10:35:02.273Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-16T10:35:02.716Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j6352640-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:94cee54eb85d08891b179bd74be48632bf6e7968f046225abb6074b4655891d0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:100m memory:256Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j6352640-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:0af23e7645c07968b9bd20f5c4a2966b34ccdaf8a05812b5eb5a3a26c53f24b5 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:ad132446-7529-417f-aff8-31d826ffb6f2 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#ad132446-7529-417f-aff8-31d826ffb6f2 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-16T10:35:02.716Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:35:02.863Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-16T10:35:03.042Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-16T10:35:03.042Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:35:03.143Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:45:05.949Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T10:45:05.949Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T10:46:05.955Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-16T10:46:06.637Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j6352640-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:94cee54eb85d08891b179bd74be48632bf6e7968f046225abb6074b4655891d0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:100m memory:256Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j6352640-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:0af23e7645c07968b9bd20f5c4a2966b34ccdaf8a05812b5eb5a3a26c53f24b5 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:ad132446-7529-417f-aff8-31d826ffb6f2 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#ad132446-7529-417f-aff8-31d826ffb6f2 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-16T10:46:06.637Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:46:07.013Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-16T10:46:07.408Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:46:07.408Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:46:07.670Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-16T10:46:08.488Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:46:08.489Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:46:08.888Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:46:09.401Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T10:46:09.747Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T10:46:09.959Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:46:10.361Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:46:10.729Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:46:11.065Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-16T10:46:11.065Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:46:11.148Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-16T10:56:12.973Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T10:56:12.973Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T10:57:12.977Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-16T10:57:13.486Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j6352640-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:94cee54eb85d08891b179bd74be48632bf6e7968f046225abb6074b4655891d0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:100m memory:256Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j6352640-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:0af23e7645c07968b9bd20f5c4a2966b34ccdaf8a05812b5eb5a3a26c53f24b5 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:ad132446-7529-417f-aff8-31d826ffb6f2 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#ad132446-7529-417f-aff8-31d826ffb6f2 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-16T10:57:13.486Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:57:13.807Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-16T10:57:14.174Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:57:14.174Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:57:14.404Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-16T10:57:15.329Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:57:15.329Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:57:15.688Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:57:15.987Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T10:57:16.345Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T10:57:16.535Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:57:16.747Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:57:16.984Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:57:17.208Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-16T10:57:17.208Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:57:17.300Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-16T11:05:02.358Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"context deadline exceeded\"\ntime=2026-09-16T11:05:02.359Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","failure_pattern":"failed to roll out Helm release: context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161384225443840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6922,"pr_state":"open","pr_sha":"87e40a07142a1457ed3f83374cc5d085c87d8f29","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T09:54:27Z","started_at":"2026-09-16T09:54:27Z","completed_at":"2026-09-16T11:57:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:40.543666754Z","timing_metadata_checked_at":"2026-09-18T15:40:40.543666754Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:40.547005266Z","region_metadata_checked_at":"2026-09-18T15:40:40.547005266Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"111ac821af981e45dac1a79c673f51cdcb4f2de606d5e5a66356c674447051f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161384225443840","occurred_at":"2026-09-16T09:54:27Z","failed_at":"e2e","signature_id":"3fd557fa1356d64ec7477ff53929400119de4eda55d74d194fe4734c675f465b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-cw6cmg9l2kxw/cluster-candidate-4-22-7hbdnb should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000599580\u003e: \n failed to create HCP cluster cluster-candidate-4-22-7hbdnb: failed waiting for cluster=\"cluster-candidate-4-22-7hbdnb\" in resourcegroup=\"rg-candidate-4-22-cw6cmg9l2kxw\" to finish creating: GET https://rp.j5443840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b7411591-3119-4eb0-9f0c-3b28443fa9a7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b7411591-3119-4eb0-9f0c-3b28443fa9a7\",\n \"name\": \"b7411591-3119-4eb0-9f0c-3b28443fa9a7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T10:44:05.206761929Z\",\n \"endTime\": \"2026-09-16T11:13:07.811131396Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 21m28s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-cw6cmg9l2kxw/cluster-candidate-4-22-7hbdnb should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000599580\u003e: \n failed to create HCP cluster cluster-candidate-4-22-7hbdnb: failed waiting for cluster=\"cluster-candidate-4-22-7hbdnb\" in resourcegroup=\"rg-candidate-4-22-cw6cmg9l2kxw\" to finish creating: GET https://rp.j5443840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b7411591-3119-4eb0-9f0c-3b28443fa9a7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b7411591-3119-4eb0-9f0c-3b28443fa9a7\",\n \"name\": \"b7411591-3119-4eb0-9f0c-3b28443fa9a7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T10:44:05.206761929Z\",\n \"endTime\": \"2026-09-16T11:13:07.811131396Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 21m28s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c5c776d5911849fdc81cb1dc51cf4228b71d2670a332d9c5abd0f80767d4ebdb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161384225443840","occurred_at":"2026-09-16T09:54:27Z","failed_at":"alert","signature_id":"ae8593e2f93d516941e79280e7827e07a691be6004452673f6399cae8b22f618","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s) (1 unknown, 2 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T10:58:27Z\n Ended: 2026-09-16T11:25:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5443840-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss2a2d61qlifdnfp6cit52qi450302e-j1l3m8c5u1f8u2n\", pod=\"router-6d5bd595f9-fx5mz\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss2a2d61qlifdnfp6cit52qi450302e-j1l3m8c5u1f8u2n/router-6d5bd595f9-fx5mz has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"df102ba53cda8c583af0cce63e38e97c4d5207a7a82c63b1c212bb09aaad76f0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161384225443840","occurred_at":"2026-09-16T09:54:27Z","failed_at":"e2e","signature_id":"49e9dc4352795e52ee09109b3c354fc6854bce4fbeed1fea1be113f6a6209aa5","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--9f6m5qx9hqkh/cluster-hs-pre-42s85j should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c9e460\u003e: \n failed to create HCP cluster cluster-hs-pre-42s85j: failed waiting for cluster=\"cluster-hs-pre-42s85j\" in resourcegroup=\"rg-hs-pre--9f6m5qx9hqkh\" to finish creating: GET https://rp.j5443840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90bc1c83-43b8-4ba3-ab12-9113f5b59f30\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90bc1c83-43b8-4ba3-ab12-9113f5b59f30\",\n \"name\": \"90bc1c83-43b8-4ba3-ab12-9113f5b59f30\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T10:44:25.183290241Z\",\n \"endTime\": \"2026-09-16T11:13:27.895359388Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m51s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--9f6m5qx9hqkh/cluster-hs-pre-42s85j should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c9e460\u003e: \n failed to create HCP cluster cluster-hs-pre-42s85j: failed waiting for cluster=\"cluster-hs-pre-42s85j\" in resourcegroup=\"rg-hs-pre--9f6m5qx9hqkh\" to finish creating: GET https://rp.j5443840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90bc1c83-43b8-4ba3-ab12-9113f5b59f30\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90bc1c83-43b8-4ba3-ab12-9113f5b59f30\",\n \"name\": \"90bc1c83-43b8-4ba3-ab12-9113f5b59f30\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T10:44:25.183290241Z\",\n \"endTime\": \"2026-09-16T11:13:27.895359388Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m51s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100153419804184576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"3adac213c3cac05a55d60f030777f894e02ca737","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T09:22:48Z","started_at":"2026-09-16T09:22:48Z","completed_at":"2026-09-16T11:33:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:05.123294408Z","timing_metadata_checked_at":"2026-09-18T15:40:05.123294408Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:05.12656262Z","region_metadata_checked_at":"2026-09-18T15:40:05.12656262Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"4a9f7e2b321e9ba9a3fdfaa1f1072c8e66069cf50dfe2299f41afeae11487d25","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100153419804184576","occurred_at":"2026-09-16T09:22:48Z","failed_at":"alert","signature_id":"3a248c85aeccd44a74b5e82dc19ae5115320308561bbe219e225440971c85f86","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T10:17:51Z\n Ended: 2026-09-16T10:41:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4184576-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss1n67ctv55qmobaikqm5f0c9f5u0la-q5h9a2h5l8m7j8d\", pod=\"router-595c8dc9bc-fnrsm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss1n67ctv55qmobaikqm5f0c9f5u0la-q5h9a2h5l8m7j8d/router-595c8dc9bc-fnrsm has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"55ce500106e4e81270f2685962846012f30234a82f044af0a65961d2e66dcc56","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100153419804184576","occurred_at":"2026-09-16T09:22:48Z","failed_at":"e2e","signature_id":"6c4e73f54e8d63150bdff29f74282ba9a9e2ff1e60f7bd58d45e6cacc3dd7869","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:185]: failed to create SRE breakglass credentials with aro-sre-pso permissions\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a7f820\u003e: \n failed to get ready session kubeconfig from /admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-breakglass-wlfqkcdjh9rm/providers/microsoft.redhatopenshift/hcpopenshiftclusters/sre-hcp-cluster/breakglass/breakglass-lbdxk/kubeconfig: timeout waiting for session to become ready (last status: {\"HostedControlPlaneAvailable\":{\"message\":\"HostedControlPlane exists but is not ready\",\"reason\":\"HostedControlPlaneNotReady\",\"status\":\"False\"},\"status\":\"Session is not ready\"})\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:185]: failed to create SRE breakglass credentials with aro-sre-pso permissions\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a7f820\u003e: \n failed to get ready session kubeconfig from /admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-breakglass-wlfqkcdjh9rm/providers/microsoft.redhatopenshift/hcpopenshiftclusters/sre-hcp-cluster/breakglass/breakglass-lbdxk/kubeconfig: timeout waiting for session to become ready (last status: {\"HostedControlPlaneAvailable\":{\"message\":\"HostedControlPlane exists but is not ready\",\"reason\":\"HostedControlPlaneNotReady\",\"status\":\"False\"},\"status\":\"Session is not ready\"})\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e9ea5f54afe3e98f1a64398661238c6fc0e342dcd1119b0d656cfccf60db80ec","failure_pattern":"breakglass session kubeconfig not ready: timeout waiting for session to become ready; HostedControlPlane exists but is not ready","search_query":"failed to get ready session kubeconfig from /admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-breakglass-wlfqkcdjh9rm/providers/microsoft.redhatopenshift/hcpopenshiftclusters/sre-hcp-cluster/breakglass/breakglass-lbdxk/kubeconfig: timeout waiting for session to become ready (last status: {\"HostedControlPlaneAvailable\":{\"message\":\"HostedControlPlane exists but is not ready\",\"reason\":\"HostedControlPlaneNotReady\",\"status\":\"False\"},\"status\":\"Session is not ready\"})"}},{"row_id":"99949cac6fa849b829e69e7dd21d3f3610faeefe87a251179a3fa334e2d38f2d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100153419804184576","occurred_at":"2026-09-16T09:22:48Z","failed_at":"e2e","signature_id":"1df8582823fde891257155971a259af217e7163cb77525129dce1bd9c178bb48","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:124]: failed to create HCP cluster \"cluster-zstream-4-22-kbmqc2\" with version 4.22.13 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00111c540\u003e: \n failed to create HCP cluster cluster-zstream-4-22-kbmqc2: failed waiting for cluster=\"cluster-zstream-4-22-kbmqc2\" in resourcegroup=\"rg-zstream-upgrade-4-22-plv5jd5xhkrz\" to finish creating: GET https://rp.j4184576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54880953-0445-4405-8334-2c8fc8586a9a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54880953-0445-4405-8334-2c8fc8586a9a\",\n \"name\": \"54880953-0445-4405-8334-2c8fc8586a9a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T10:04:06.201723402Z\",\n \"endTime\": \"2026-09-16T10:33:14.400302806Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.13 is Partial (want Completed), started 21m59s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:124]: failed to create HCP cluster \"cluster-zstream-4-22-kbmqc2\" with version 4.22.13 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00111c540\u003e: \n failed to create HCP cluster cluster-zstream-4-22-kbmqc2: failed waiting for cluster=\"cluster-zstream-4-22-kbmqc2\" in resourcegroup=\"rg-zstream-upgrade-4-22-plv5jd5xhkrz\" to finish creating: GET https://rp.j4184576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54880953-0445-4405-8334-2c8fc8586a9a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54880953-0445-4405-8334-2c8fc8586a9a\",\n \"name\": \"54880953-0445-4405-8334-2c8fc8586a9a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T10:04:06.201723402Z\",\n \"endTime\": \"2026-09-16T10:33:14.400302806Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.13 is Partial (want Completed), started 21m59s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b972512be901489da0ed65f6b171331b85abe2f1ba07ca9270f63a91d6a701e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100153419804184576","occurred_at":"2026-09-16T09:22:48Z","failed_at":"alert","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T10:20:03Z\n Ended: 2026-09-16T10:35:01Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4184576-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","e9ea5f54afe3e98f1a64398661238c6fc0e342dcd1119b0d656cfccf60db80ec"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100148319702487040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6894,"pr_state":"closed","pr_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","final_merged_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-16T09:02:32Z","started_at":"2026-09-16T09:02:32Z","completed_at":"2026-09-16T11:10:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:33.284955738Z","timing_metadata_checked_at":"2026-09-18T15:40:33.284955738Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:33.289176753Z","region_metadata_checked_at":"2026-09-18T15:40:33.289176753Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100147953778823168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5846,"pr_state":"closed","pr_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","final_merged_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T09:00:43Z","started_at":"2026-09-16T09:00:43Z","completed_at":"2026-09-16T11:07:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:03.914025356Z","timing_metadata_checked_at":"2026-09-18T15:40:03.914025356Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:03.918881174Z","region_metadata_checked_at":"2026-09-18T15:40:03.918881174Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"3612a9cee33afc96eae9e9c7d7548b955ef1c96584f1a047c8d587a33e08b8c2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100147953778823168","occurred_at":"2026-09-16T09:00:43Z","failed_at":"e2e","signature_id":"8fc547e896d5e55d734d3ddd67ff9ad51e9b0658292885c96f9396cffb7b85a2","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001110400\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-4p7ps7sph7f6\" to finish creating: GET https://rp.j8823168.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/64e98671-810b-4be1-b0de-33ed4ed7b43e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/64e98671-810b-4be1-b0de-33ed4ed7b43e\",\n \"name\": \"64e98671-810b-4be1-b0de-33ed4ed7b43e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:51:36.009109937Z\",\n \"endTime\": \"2026-09-16T10:20:38.775019606Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 24m10s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001110400\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-4p7ps7sph7f6\" to finish creating: GET https://rp.j8823168.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/64e98671-810b-4be1-b0de-33ed4ed7b43e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/64e98671-810b-4be1-b0de-33ed4ed7b43e\",\n \"name\": \"64e98671-810b-4be1-b0de-33ed4ed7b43e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:51:36.009109937Z\",\n \"endTime\": \"2026-09-16T10:20:38.775019606Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 24m10s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3c69a66f64a827795dbbb8a38c8a812b473e61a61bf651618aac40bb501c155e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100147953778823168","occurred_at":"2026-09-16T09:00:43Z","failed_at":"e2e","signature_id":"d15fe82d34504ca3d38710b46e1d54bc220f9fcfedc628bca6bbc3cb0cd4dd6a","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ac4380\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-g8hsmlndk2gz\" to finish creating: GET https://rp.j8823168.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02861d9b-c33b-4f27-b90a-22b7a7e4608e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02861d9b-c33b-4f27-b90a-22b7a7e4608e\",\n \"name\": \"02861d9b-c33b-4f27-b90a-22b7a7e4608e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:50:12.871173483Z\",\n \"endTime\": \"2026-09-16T10:19:18.822990124Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m37s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ac4380\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-g8hsmlndk2gz\" to finish creating: GET https://rp.j8823168.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02861d9b-c33b-4f27-b90a-22b7a7e4608e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02861d9b-c33b-4f27-b90a-22b7a7e4608e\",\n \"name\": \"02861d9b-c33b-4f27-b90a-22b7a7e4608e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:50:12.871173483Z\",\n \"endTime\": \"2026-09-16T10:19:18.822990124Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m37s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7c290b2b24b5613af462969adb4cf146a337333b5b1cb0618ba200118af35a44","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100147953778823168","occurred_at":"2026-09-16T09:00:43Z","failed_at":"e2e","signature_id":"5aa313310cb85b534b6ae9ece7e869257889473057f8f6e20312e1cf45aa795f","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b242a0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-zvr2lfbsdzcr\" to finish creating: GET https://rp.j8823168.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00bdd26c-d6ec-4e17-85fd-75e0475b1ea2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00bdd26c-d6ec-4e17-85fd-75e0475b1ea2\",\n \"name\": \"00bdd26c-d6ec-4e17-85fd-75e0475b1ea2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:49:28.418802254Z\",\n \"endTime\": \"2026-09-16T10:18:28.723004291Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m29s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b242a0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-zvr2lfbsdzcr\" to finish creating: GET https://rp.j8823168.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00bdd26c-d6ec-4e17-85fd-75e0475b1ea2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00bdd26c-d6ec-4e17-85fd-75e0475b1ea2\",\n \"name\": \"00bdd26c-d6ec-4e17-85fd-75e0475b1ea2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:49:28.418802254Z\",\n \"endTime\": \"2026-09-16T10:18:28.723004291Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m29s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"807f041e88d11c48b001f247fbfab37c2d5acc4b68bac4954487fc4c93c3a8a5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100147953778823168","occurred_at":"2026-09-16T09:00:43Z","failed_at":"e2e","signature_id":"5a06bfe3d19814f7c0a59d9e13f30fd4c6f1a6a7313da746fc7cd3c44a2f7939","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000830a80\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-v7jwmpn72lgg\" to finish creating: GET https://rp.j8823168.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bb850b73-14fc-422c-9a1f-30bf8f513961\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bb850b73-14fc-422c-9a1f-30bf8f513961\",\n \"name\": \"bb850b73-14fc-422c-9a1f-30bf8f513961\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:50:53.773970906Z\",\n \"endTime\": \"2026-09-16T10:19:58.777848052Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m42s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000830a80\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-v7jwmpn72lgg\" to finish creating: GET https://rp.j8823168.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bb850b73-14fc-422c-9a1f-30bf8f513961\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bb850b73-14fc-422c-9a1f-30bf8f513961\",\n \"name\": \"bb850b73-14fc-422c-9a1f-30bf8f513961\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:50:53.773970906Z\",\n \"endTime\": \"2026-09-16T10:19:58.777848052Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m42s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8ad9969e832960912dc64956d8be14dc1fe1f84e841b4c6b06d161b6cd884a73","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100147953778823168","occurred_at":"2026-09-16T09:00:43Z","failed_at":"e2e","signature_id":"b19cd1523bca64c96d563324ee6475a23c1199cf8856040f6afdad3121dbc86f","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--kj428xsfb48q/cluster-hs-pre-bf84l4 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00115c700\u003e: \n failed to create HCP cluster cluster-hs-pre-bf84l4: failed waiting for cluster=\"cluster-hs-pre-bf84l4\" in resourcegroup=\"rg-hs-pre--kj428xsfb48q\" to finish creating: GET https://rp.j8823168.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aea5d780-ec83-4b09-8960-8b31d26d83b1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aea5d780-ec83-4b09-8960-8b31d26d83b1\",\n \"name\": \"aea5d780-ec83-4b09-8960-8b31d26d83b1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:49:11.962303449Z\",\n \"endTime\": \"2026-09-16T10:18:18.875942327Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m21s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--kj428xsfb48q/cluster-hs-pre-bf84l4 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00115c700\u003e: \n failed to create HCP cluster cluster-hs-pre-bf84l4: failed waiting for cluster=\"cluster-hs-pre-bf84l4\" in resourcegroup=\"rg-hs-pre--kj428xsfb48q\" to finish creating: GET https://rp.j8823168.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aea5d780-ec83-4b09-8960-8b31d26d83b1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aea5d780-ec83-4b09-8960-8b31d26d83b1\",\n \"name\": \"aea5d780-ec83-4b09-8960-8b31d26d83b1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:49:11.962303449Z\",\n \"endTime\": \"2026-09-16T10:18:18.875942327Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m21s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6923/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100146883333722112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6923,"pr_state":"closed","pr_sha":"9e6650394c80bca242363572272841073766ea22","final_merged_sha":"9e6650394c80bca242363572272841073766ea22","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-16T08:56:50Z","started_at":"2026-09-16T08:56:50Z","completed_at":"2026-09-16T11:16:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:41.3065053Z","timing_metadata_checked_at":"2026-09-18T15:40:41.3065053Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:41.309603711Z","region_metadata_checked_at":"2026-09-18T15:40:41.309603711Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6889,"pr_state":"open","pr_sha":"ab7c1a34a27ce634138b2690ee7e630dacfd6f63","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T08:35:58Z","started_at":"2026-09-16T08:35:58Z","completed_at":"2026-09-16T11:04:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:32.673664339Z","timing_metadata_checked_at":"2026-09-18T15:40:32.673664339Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:32.677442053Z","region_metadata_checked_at":"2026-09-18T15:40:32.677442053Z"},"failed_at":["alert","e2e"],"failed_test_count":18,"occurrences":[{"row_id":"15901106365f4f2a7b41ff434b3008b7cdabb4cb0d41a96c4f43875eb1a8b4f5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","failed_at":"e2e","signature_id":"d539c69fd361c2a857c348ec75ef524c1710267ecef6f193ceb123bc3e8bce18","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00168c138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-px8wfxs26nnj\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cd68e491-5165-4d43-8cd3-0c235f3ba9c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cd68e491-5165-4d43-8cd3-0c235f3ba9c5\",\n \"name\": \"cd68e491-5165-4d43-8cd3-0c235f3ba9c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:40:28.518309442Z\",\n \"endTime\": \"2026-09-16T10:04:32.390225559Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00168c138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-px8wfxs26nnj\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cd68e491-5165-4d43-8cd3-0c235f3ba9c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cd68e491-5165-4d43-8cd3-0c235f3ba9c5\",\n \"name\": \"cd68e491-5165-4d43-8cd3-0c235f3ba9c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:40:28.518309442Z\",\n \"endTime\": \"2026-09-16T10:04:32.390225559Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40f7a558a9f14e3ad84a0dc62150fb9e1dcec198cbfa1c00cbbb104cbc63f3a7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1cb3f3d55e306a03b0be1e8c2a9a9987339a694716ecfa1c30d793ce9264fb24","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","failed_at":"e2e","signature_id":"3f36db93067dda2082a5ba2aa4539a20530e919b951302455f99df670738d8ff","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-9bqqftbhhkhb/cluster-candidate-5-0-rgjngf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f68580\u003e: \n failed to create HCP cluster cluster-candidate-5-0-rgjngf: failed waiting for cluster=\"cluster-candidate-5-0-rgjngf\" in resourcegroup=\"rg-candidate-5-0-9bqqftbhhkhb\" to finish creating: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3baf4e11-4b0e-4700-b113-da3918d872d5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3baf4e11-4b0e-4700-b113-da3918d872d5\",\n \"name\": \"3baf4e11-4b0e-4700-b113-da3918d872d5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:21:27.799687923Z\",\n \"endTime\": \"2026-09-16T09:50:32.506640388Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.2 is Partial (want Completed), started 18m15s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-9bqqftbhhkhb/cluster-candidate-5-0-rgjngf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f68580\u003e: \n failed to create HCP cluster cluster-candidate-5-0-rgjngf: failed waiting for cluster=\"cluster-candidate-5-0-rgjngf\" in resourcegroup=\"rg-candidate-5-0-9bqqftbhhkhb\" to finish creating: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3baf4e11-4b0e-4700-b113-da3918d872d5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3baf4e11-4b0e-4700-b113-da3918d872d5\",\n \"name\": \"3baf4e11-4b0e-4700-b113-da3918d872d5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:21:27.799687923Z\",\n \"endTime\": \"2026-09-16T09:50:32.506640388Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.2 is Partial (want Completed), started 18m15s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"36388d0dfbc533cbd8bd55d1c52d93a9adedd97c4965b80e551bdf0ffdaf88db","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","failed_at":"e2e","signature_id":"70756564bd668b5819cf49c7d07af12de78c4ca05f5acb0024a436a269a2a249","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0010be048\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-lq45ld6h888d\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76270801-127d-4225-97dc-06e57322bf5a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76270801-127d-4225-97dc-06e57322bf5a\",\n \"name\": \"76270801-127d-4225-97dc-06e57322bf5a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:37:08.667361714Z\",\n \"endTime\": \"2026-09-16T10:01:12.766789046Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss13j30utlk9cm4u8n21tm903korv38 still exists (deletion dispatched at 2026-09-16T09:37:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0010be048\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-lq45ld6h888d\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76270801-127d-4225-97dc-06e57322bf5a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76270801-127d-4225-97dc-06e57322bf5a\",\n \"name\": \"76270801-127d-4225-97dc-06e57322bf5a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:37:08.667361714Z\",\n \"endTime\": \"2026-09-16T10:01:12.766789046Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss13j30utlk9cm4u8n21tm903korv38 still exists (deletion dispatched at 2026-09-16T09:37:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f7e85048bd6764cc5537fb876507c8173ed1c8737be07b8c67fd9fcfdae58024","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3b1dd11b811dce36b62adce1113013f55433536382edb9b9a0529bbfcf912385","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","failed_at":"e2e","signature_id":"ab25cdf796ca0e23837f6fcc81a18f44c0e9d3bfb187f38beeb217788731b679","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c2c360\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-rmqvg72srtqc\" to finish creating: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/88e478a1-228a-4fba-bfa8-f2447c215df2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/88e478a1-228a-4fba-bfa8-f2447c215df2\",\n \"name\": \"88e478a1-228a-4fba-bfa8-f2447c215df2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:21:39.030244884Z\",\n \"endTime\": \"2026-09-16T09:50:42.360079347Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 18m27s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c2c360\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-rmqvg72srtqc\" to finish creating: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/88e478a1-228a-4fba-bfa8-f2447c215df2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/88e478a1-228a-4fba-bfa8-f2447c215df2\",\n \"name\": \"88e478a1-228a-4fba-bfa8-f2447c215df2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:21:39.030244884Z\",\n \"endTime\": \"2026-09-16T09:50:42.360079347Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 18m27s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"633e5b1c21818ac196bb9aead6154fb511190d6d054967660ee14de6aa9a2e49","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","failed_at":"alert","signature_id":"278a7d810c61d983530553cb6ff8366e5a4d79b77f9c15651d9ebcb3dbeb9e4e","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 20 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T10:35:42Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5539456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.187:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5539456/providers/microsoft.monitor/accounts/hcps-j5539456\", namespace=\"ocm-arohcpci01-2ss1371bjtto4nf5m6o0pid4pjvle6qc-w3u2s1n8u7b7c4o\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-x7c5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ss1371bjtto4nf5m6o0pid4pjvle6qc-w3u2s1n8u7b7c4o/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-16T10:35:42Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5539456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.187:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5539456/providers/microsoft.monitor/accounts/hcps-j5539456\", namespace=\"ocm-arohcpci01-2ss12vvsmbd0thdp0f88kc41u6pvi7c9-y7r0c6f2g8v3k8g\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-x7c5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ss12vvsmbd0thdp0f88kc41u6pvi7c9-y7r0c6f2g8v3k8g/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-16T10:35:44Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5539456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.187:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5539456/providers/microsoft.monitor/accounts/hcps-j5539456\", namespace=\"ocm-arohcpci01-2ss1371bjtto4nf5m6o0pid4pjvle6qc-w3u2s1n8u7b7c4o\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-x7c5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ss1371bjtto4nf5m6o0pid4pjvle6qc-w3u2s1n8u7b7c4o/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-16T10:35:44Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5539456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.187:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5539456/providers/microsoft.monitor/accounts/hcps-j5539456\", namespace=\"ocm-arohcpci01-2ss12vvsmbd0thdp0f88kc41u6pvi7c9-y7r0c6f2g8v3k8g\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-x7c5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ss12vvsmbd0thdp0f88kc41u6pvi7c9-y7r0c6f2g8v3k8g/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-09-16T10:35:44Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5539456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"ingress-operator\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.187:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5539456/providers/microsoft.monitor/accounts/hcps-j5539456\", namespace=\"ocm-arohcpci01-2ss12t13cvqgvb69duuok3m0c1if4g29-q1k5h2w4a4a1f5u\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-x7c5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ss12t13cvqgvb69duuok3m0c1if4g29-q1k5h2w4a4a1f5u/ingress-operator has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-09-16T10:35:44Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5539456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"dns-operator\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.187:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5539456/providers/microsoft.monitor/accounts/hcps-j5539456\", namespace=\"ocm-arohcpci01-2ss12t13cvqgvb69duuok3m0c1if4g29-q1k5h2w4a4a1f5u\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-x7c5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ss12t13cvqgvb69duuok3m0c1if4g29-q1k5h2w4a4a1f5u/dns-operator has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-09-16T10:36:44Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5539456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"ingress-operator\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.187:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5539456/providers/microsoft.monitor/accounts/hcps-j5539456\", namespace=\"ocm-arohcpci01-2ss12t13cvqgvb69duuok3m0c1if4g29-q1k5h2w4a4a1f5u\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-x7c5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ss12t13cvqgvb69duuok3m0c1if4g29-q1k5h2w4a4a1f5u/ingress-operator has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n State: Fired (not resolved)\n Started: 2026-09-16T10:36:44Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5539456-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.154:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5539456/providers/microsoft.monitor/accounts/hcps-j5539456\", namespace=\"ocm-arohcpci01-2ss13j2abs333rpsqenh1eeh8ha7bprr-e2y3l7s9r3u6x1b\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qmqb6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ss13j2abs333rpsqenh1eeh8ha7bprr-e2y3l7s9r3u6x1b/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n State: Fired (not resolved)\n Started: 2026-09-16T10:36:44Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5539456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.187:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5539456/providers/microsoft.monitor/accounts/hcps-j5539456\", namespace=\"ocm-arohcpci01-2ss1371bjtto4nf5m6o0pid4pjvle6qc-w3u2s1n8u7b7c4o\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-x7c5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ss1371bjtto4nf5m6o0pid4pjvle6qc-w3u2s1n8u7b7c4o/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n State: Fired (not resolved)\n Started: 2026-09-16T10:36:44Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5539456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.187:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5539456/providers/microsoft.monitor/accounts/hcps-j5539456\", namespace=\"ocm-arohcpci01-2ss12vvsmbd0thdp0f88kc41u6pvi7c9-y7r0c6f2g8v3k8g\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-x7c5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ss12vvsmbd0thdp0f88kc41u6pvi7c9-y7r0c6f2g8v3k8g/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n State: Fired (not resolved)\n Started: 2026-09-16T10:36:44Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5539456-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.154:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5539456/providers/microsoft.monitor/accounts/hcps-j5539456\", namespace=\"ocm-arohcpci01-2ss1361u3o4ev8vfck5hk2a42dp30j3q-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qmqb6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ss1361u3o4ev8vfck5hk2a42dp30j3q-basic-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n State: Fired (not resolved)\n Started: 2026-09-16T10:36:44Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5539456-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.154:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5539456/providers/microsoft.monitor/accounts/hcps-j5539456\", namespace=\"ocm-arohcpci01-2ss1361u3o4ev8vfck5hk2a42dp30j3q-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qmqb6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ss1361u3o4ev8vfck5hk2a42dp30j3q-basic-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n State: Fired (not resolved)\n Started: 2026-09-16T10:36:44Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5539456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.187:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5539456/providers/microsoft.monitor/accounts/hcps-j5539456\", namespace=\"ocm-arohcpci01-2ss1371bjtto4nf5m6o0pid4pjvle6qc-w3u2s1n8u7b7c4o\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-x7c5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ss1371bjtto4nf5m6o0pid4pjvle6qc-w3u2s1n8u7b7c4o/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 14:\n State: Fired (not resolved)\n Started: 2026-09-16T10:36:44Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5539456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.187:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5539456/providers/microsoft.monitor/accounts/hcps-j5539456\", namespace=\"ocm-arohcpci01-2ss12vvsmbd0thdp0f88kc41u6pvi7c9-y7r0c6f2g8v3k8g\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-x7c5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ss12vvsmbd0thdp0f88kc41u6pvi7c9-y7r0c6f2g8v3k8g/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 15:\n State: Fired (not resolved)\n Started: 2026-09-16T10:36:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5539456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"dns-operator\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.187:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5539456/providers/microsoft.monitor/accounts/hcps-j5539456\", namespace=\"ocm-arohcpci01-2ss12t13cvqgvb69duuok3m0c1if4g29-q1k5h2w4a4a1f5u\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-x7c5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ss12t13cvqgvb69duuok3m0c1if4g29-q1k5h2w4a4a1f5u/dns-operator has not matched the expected number of replicas for longer than 30 minutes.\nFiring 16:\n State: Fired (not resolved)\n Started: 2026-09-16T10:36:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5539456-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.154:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5539456/providers/microsoft.monitor/accounts/hcps-j5539456\", namespace=\"ocm-arohcpci01-2ss13j2abs333rpsqenh1eeh8ha7bprr-e2y3l7s9r3u6x1b\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qmqb6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ss13j2abs333rpsqenh1eeh8ha7bprr-e2y3l7s9r3u6x1b/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 17:\n State: Fired (not resolved)\n Started: 2026-09-16T10:44:44Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5539456-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.154:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5539456/providers/microsoft.monitor/accounts/hcps-j5539456\", namespace=\"ocm-arohcpci01-2ss1361u3o4ev8vfck5hk2a42dp30j3q-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qmqb6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ss1361u3o4ev8vfck5hk2a42dp30j3q-basic-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 18:\n State: Fired (not resolved)\n Started: 2026-09-16T10:44:44Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5539456-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.154:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5539456/providers/microsoft.monitor/accounts/hcps-j5539456\", namespace=\"ocm-arohcpci01-2ss1361u3o4ev8vfck5hk2a42dp30j3q-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qmqb6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ss1361u3o4ev8vfck5hk2a42dp30j3q-basic-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 19:\n State: Fired (not resolved)\n Started: 2026-09-16T10:44:44Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5539456-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.154:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5539456/providers/microsoft.monitor/accounts/hcps-j5539456\", namespace=\"ocm-arohcpci01-2ss13j2abs333rpsqenh1eeh8ha7bprr-e2y3l7s9r3u6x1b\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qmqb6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ss13j2abs333rpsqenh1eeh8ha7bprr-e2y3l7s9r3u6x1b/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 20:\n State: Fired (not resolved)\n Started: 2026-09-16T10:44:44Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5539456-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.154:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5539456/providers/microsoft.monitor/accounts/hcps-j5539456\", namespace=\"ocm-arohcpci01-2ss13j2abs333rpsqenh1eeh8ha7bprr-e2y3l7s9r3u6x1b\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qmqb6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ss13j2abs333rpsqenh1eeh8ha7bprr-e2y3l7s9r3u6x1b/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"6511e0650053a3daf86d86413f1b3b82744295417e532388953da426d47d70a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","failed_at":"e2e","signature_id":"5a758cc0e5fd4d4f0a2773989c77d2a7c459fe910fa5486bed83da0d068c3f31","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011720a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-5-0-4cfzlz\" in resourcegroup=\"rg-zstream-upgrade-5-0-2bmmvg6nm4m6\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a49e79c0-b564-4703-ad68-3e8fbb555ee7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a49e79c0-b564-4703-ad68-3e8fbb555ee7\",\n \"name\": \"a49e79c0-b564-4703-ad68-3e8fbb555ee7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:52:39.028633437Z\",\n \"endTime\": \"2026-09-16T10:16:42.563449333Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss136411ov65aeqqn1ut2t4iprrujv9 still exists (deletion dispatched at 2026-09-16T09:52:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011720a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-5-0-4cfzlz\" in resourcegroup=\"rg-zstream-upgrade-5-0-2bmmvg6nm4m6\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a49e79c0-b564-4703-ad68-3e8fbb555ee7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a49e79c0-b564-4703-ad68-3e8fbb555ee7\",\n \"name\": \"a49e79c0-b564-4703-ad68-3e8fbb555ee7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:52:39.028633437Z\",\n \"endTime\": \"2026-09-16T10:16:42.563449333Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss136411ov65aeqqn1ut2t4iprrujv9 still exists (deletion dispatched at 2026-09-16T09:52:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f7e85048bd6764cc5537fb876507c8173ed1c8737be07b8c67fd9fcfdae58024","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6b2752298693ab9c40b615c373b77d05936989f73eada61f0c53f27196de1443","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","failed_at":"e2e","signature_id":"ba6d9f57396d35aba73dd64e7c229f8d7cbb408e58e239c7c2b2065df4e28656","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000bfe3a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-wgdjtrkdh4nq\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d757bcf9-a836-4d4b-93e9-18f5495774b1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d757bcf9-a836-4d4b-93e9-18f5495774b1\",\n \"name\": \"d757bcf9-a836-4d4b-93e9-18f5495774b1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:47:47.203138976Z\",\n \"endTime\": \"2026-09-16T10:11:52.474449775Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000bfe3a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-wgdjtrkdh4nq\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d757bcf9-a836-4d4b-93e9-18f5495774b1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d757bcf9-a836-4d4b-93e9-18f5495774b1\",\n \"name\": \"d757bcf9-a836-4d4b-93e9-18f5495774b1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:47:47.203138976Z\",\n \"endTime\": \"2026-09-16T10:11:52.474449775Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40f7a558a9f14e3ad84a0dc62150fb9e1dcec198cbfa1c00cbbb104cbc63f3a7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7cdb65b7f28fe198c5d5d9684e4786938645c6f3cfe7800077394c9de40028d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","failed_at":"e2e","signature_id":"6ba2d7f149e18d77de06d3b79ce4ecb80b3fb20da43bf259cd0570583e9d7af6","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0008820c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-hz87cjqncdvb\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8e715185-33d5-494b-976c-25b2ab261cc3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8e715185-33d5-494b-976c-25b2ab261cc3\",\n \"name\": \"8e715185-33d5-494b-976c-25b2ab261cc3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:35:58.536453378Z\",\n \"endTime\": \"2026-09-16T10:00:02.536562643Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss1371bjtto4nf5m6o0pid4pjvle6qc still exists (deletion dispatched at 2026-09-16T09:35:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0008820c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-hz87cjqncdvb\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8e715185-33d5-494b-976c-25b2ab261cc3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8e715185-33d5-494b-976c-25b2ab261cc3\",\n \"name\": \"8e715185-33d5-494b-976c-25b2ab261cc3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:35:58.536453378Z\",\n \"endTime\": \"2026-09-16T10:00:02.536562643Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss1371bjtto4nf5m6o0pid4pjvle6qc still exists (deletion dispatched at 2026-09-16T09:35:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"984c1a7bd9f3fe0c34a67f23a4fe48c9accfda2535e107053b85021dda91a806","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","failed_at":"e2e","signature_id":"94776e25785ac46d0a71172bc6722ed5b40d2bea8cb9370d89cdf09f1fa5c805","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000706300\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-h546tchcqt6d\" to finish creating: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bee6c146-3516-4eb7-bb5d-77be8886d51f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bee6c146-3516-4eb7-bb5d-77be8886d51f\",\n \"name\": \"bee6c146-3516-4eb7-bb5d-77be8886d51f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:23:24.300104342Z\",\n \"endTime\": \"2026-09-16T09:52:32.410749237Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 20m44s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000706300\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-h546tchcqt6d\" to finish creating: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bee6c146-3516-4eb7-bb5d-77be8886d51f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bee6c146-3516-4eb7-bb5d-77be8886d51f\",\n \"name\": \"bee6c146-3516-4eb7-bb5d-77be8886d51f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:23:24.300104342Z\",\n \"endTime\": \"2026-09-16T09:52:32.410749237Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 20m44s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b8f0b1505907815aa0e6bf1539683ddeedc01cfc710f67ac6884358842a3c3ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","failed_at":"e2e","signature_id":"9635d65ddbc8d8183553acfbe1c7e3ebf7cfb49f71fb8721193391b355cf2fbd","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0018ba120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-5p7pgd\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-bd9zhbrlv7hq\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83958f5a-6d08-4efa-bef6-3bc600801633\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83958f5a-6d08-4efa-bef6-3bc600801633\",\n \"name\": \"83958f5a-6d08-4efa-bef6-3bc600801633\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:51:01.642991125Z\",\n \"endTime\": \"2026-09-16T10:15:02.48831324Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0018ba120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-5p7pgd\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-bd9zhbrlv7hq\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83958f5a-6d08-4efa-bef6-3bc600801633\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83958f5a-6d08-4efa-bef6-3bc600801633\",\n \"name\": \"83958f5a-6d08-4efa-bef6-3bc600801633\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:51:01.642991125Z\",\n \"endTime\": \"2026-09-16T10:15:02.48831324Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40f7a558a9f14e3ad84a0dc62150fb9e1dcec198cbfa1c00cbbb104cbc63f3a7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c0afe9e723d1d73abf2e26053ef23067074dbfe86ddb3657f8f41fd2194eebd8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","failed_at":"alert","signature_id":"cf50089acb0846a23f15800cf9d1588e5a42034409429bcfeb7f8b4bdce19e86","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T09:37:46Z\n Ended: 2026-09-16T10:02:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5539456-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss143dkb8crn6be34gbib5ir298a46p-cilium-cluster\", pod=\"router-d8b5b89dd-9h8gj\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss143dkb8crn6be34gbib5ir298a46p-cilium-cluster/router-d8b5b89dd-9h8gj has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"c6b84a08bcdfcc7b4d4c579a317da000f0d20c7ebee40726b704964bc52ae417","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","failed_at":"e2e","signature_id":"697bd9b6305361c1b5307d24f6145b0b596cf49460c3ec8e96a3eae19ea908a8","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001512108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-mwjj2q\" in resourcegroup=\"admin-credential-lifecycle-test-q4n7ckbgk7dt\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5124f997-7212-409c-99ae-fbd89a51e2f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5124f997-7212-409c-99ae-fbd89a51e2f4\",\n \"name\": \"5124f997-7212-409c-99ae-fbd89a51e2f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:44:00.208113506Z\",\n \"endTime\": \"2026-09-16T10:08:02.509863824Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss12vvsmbd0thdp0f88kc41u6pvi7c9 still exists (deletion dispatched at 2026-09-16T09:44:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001512108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-mwjj2q\" in resourcegroup=\"admin-credential-lifecycle-test-q4n7ckbgk7dt\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5124f997-7212-409c-99ae-fbd89a51e2f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5124f997-7212-409c-99ae-fbd89a51e2f4\",\n \"name\": \"5124f997-7212-409c-99ae-fbd89a51e2f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:44:00.208113506Z\",\n \"endTime\": \"2026-09-16T10:08:02.509863824Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss12vvsmbd0thdp0f88kc41u6pvi7c9 still exists (deletion dispatched at 2026-09-16T09:44:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ccc08ff908641611bb4d45ec68efed64acb91bcab510c27c5a9f9c2ba98fb34c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","failed_at":"e2e","signature_id":"9410511803cbf1e3c159d5decef593399c3f8d02958271b7b1f469d232f45e6e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000012f78\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-g9bvp2\" in resourcegroup=\"rg-candidate-4-20-pmdbfsk6v6zd\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/834c2939-b42e-48bd-b340-960d8826a429\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/834c2939-b42e-48bd-b340-960d8826a429\",\n \"name\": \"834c2939-b42e-48bd-b340-960d8826a429\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:49:50.424611242Z\",\n \"endTime\": \"2026-09-16T10:13:52.492147666Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000012f78\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-g9bvp2\" in resourcegroup=\"rg-candidate-4-20-pmdbfsk6v6zd\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/834c2939-b42e-48bd-b340-960d8826a429\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/834c2939-b42e-48bd-b340-960d8826a429\",\n \"name\": \"834c2939-b42e-48bd-b340-960d8826a429\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:49:50.424611242Z\",\n \"endTime\": \"2026-09-16T10:13:52.492147666Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40f7a558a9f14e3ad84a0dc62150fb9e1dcec198cbfa1c00cbbb104cbc63f3a7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d0b998a53112cb1315eda5461b6e99be391b6aa986543dfa9bd94a7a9d4429eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","failed_at":"e2e","signature_id":"18c69429807d38cfdd701e251877ce552b6dddaac55af8cb470eb1279a95b526","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0028a4438\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-j5zkl24x6bwt\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ecb9f85-876c-467d-91c9-980ce11c9827\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ecb9f85-876c-467d-91c9-980ce11c9827\",\n \"name\": \"5ecb9f85-876c-467d-91c9-980ce11c9827\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:49:19.070350042Z\",\n \"endTime\": \"2026-09-16T10:13:22.487477443Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0028a4438\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-j5zkl24x6bwt\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ecb9f85-876c-467d-91c9-980ce11c9827\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ecb9f85-876c-467d-91c9-980ce11c9827\",\n \"name\": \"5ecb9f85-876c-467d-91c9-980ce11c9827\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:49:19.070350042Z\",\n \"endTime\": \"2026-09-16T10:13:22.487477443Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40f7a558a9f14e3ad84a0dc62150fb9e1dcec198cbfa1c00cbbb104cbc63f3a7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e8505084e84f4af15a70078aed3b6d7f417b52372011aba25c4bf98c5bc5ae88","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","failed_at":"e2e","signature_id":"3ce6a891cb0d07c7ce901cd0ee33f19037e602ebe5639fc8eb520176e22be57f","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001580078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dfbjv854d6w5\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d99ec0f0-e1d1-41b0-b753-9ba0ef65c4b3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d99ec0f0-e1d1-41b0-b753-9ba0ef65c4b3\",\n \"name\": \"d99ec0f0-e1d1-41b0-b753-9ba0ef65c4b3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:36:31.257109777Z\",\n \"endTime\": \"2026-09-16T10:00:32.532474531Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss1375769r7k6ubu137jlbqtq9j35it still exists (deletion dispatched at 2026-09-16T09:36:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001580078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dfbjv854d6w5\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d99ec0f0-e1d1-41b0-b753-9ba0ef65c4b3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d99ec0f0-e1d1-41b0-b753-9ba0ef65c4b3\",\n \"name\": \"d99ec0f0-e1d1-41b0-b753-9ba0ef65c4b3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:36:31.257109777Z\",\n \"endTime\": \"2026-09-16T10:00:32.532474531Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss1375769r7k6ubu137jlbqtq9j35it still exists (deletion dispatched at 2026-09-16T09:36:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f7e85048bd6764cc5537fb876507c8173ed1c8737be07b8c67fd9fcfdae58024","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"eed97c05833212768b126c68bd46a3da5ea04c75fc816d455817d1c4d7dea552","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","failed_at":"e2e","signature_id":"ff4cffb5ab05b8596815bef0f9113f08c5ccdefbdae02ac3f027a27a7d4486ea","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0019ec138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-gql2drdp7tjr\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/92c64afb-54ba-4f5e-a0ce-1a57e90c2b0b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/92c64afb-54ba-4f5e-a0ce-1a57e90c2b0b\",\n \"name\": \"92c64afb-54ba-4f5e-a0ce-1a57e90c2b0b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:34:43.828694597Z\",\n \"endTime\": \"2026-09-16T09:58:52.519202639Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss1361u3o4ev8vfck5hk2a42dp30j3q still exists (deletion dispatched at 2026-09-16T09:34:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0019ec138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-gql2drdp7tjr\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/92c64afb-54ba-4f5e-a0ce-1a57e90c2b0b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/92c64afb-54ba-4f5e-a0ce-1a57e90c2b0b\",\n \"name\": \"92c64afb-54ba-4f5e-a0ce-1a57e90c2b0b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:34:43.828694597Z\",\n \"endTime\": \"2026-09-16T09:58:52.519202639Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss1361u3o4ev8vfck5hk2a42dp30j3q still exists (deletion dispatched at 2026-09-16T09:34:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f4a8c9b7d6d4364871e17a622194e9725e7a8b3437dbafac52947a51c4885422","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","failed_at":"e2e","signature_id":"f25ee5738903cfa5e80de14eb219fe7a6033dded2da21704c137087674937bf8","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000f92288\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-djgr48qrs9vr\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a17df9ed-a5b1-4a4c-b59d-0c40cfdd60e7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a17df9ed-a5b1-4a4c-b59d-0c40cfdd60e7\",\n \"name\": \"a17df9ed-a5b1-4a4c-b59d-0c40cfdd60e7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:36:21.892022588Z\",\n \"endTime\": \"2026-09-16T10:00:22.50023585Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss12t13cvqgvb69duuok3m0c1if4g29 still exists (deletion dispatched at 2026-09-16T09:36:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000f92288\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-djgr48qrs9vr\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a17df9ed-a5b1-4a4c-b59d-0c40cfdd60e7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a17df9ed-a5b1-4a4c-b59d-0c40cfdd60e7\",\n \"name\": \"a17df9ed-a5b1-4a4c-b59d-0c40cfdd60e7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:36:21.892022588Z\",\n \"endTime\": \"2026-09-16T10:00:22.50023585Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss12t13cvqgvb69duuok3m0c1if4g29 still exists (deletion dispatched at 2026-09-16T09:36:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a9f77fe91a65ae001825b7e195168037cff1cc1fd2f1b2a967813a5f71db56d2","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down pod network instances, pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ff8d901e0ae45e2e248948af546f1597a6c15dffec9e85c768ad41b08380b3d6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","failed_at":"e2e","signature_id":"232d76b4c8604eadbe09c019eea03e4b35e8f44b8a88fbcca5afcc985e967ce4","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00153e0f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-9dwsgxpprw8c\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e39e5c6-0d40-4bf6-9634-0b867d3cf17c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e39e5c6-0d40-4bf6-9634-0b867d3cf17c\",\n \"name\": \"9e39e5c6-0d40-4bf6-9634-0b867d3cf17c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:44:48.885828252Z\",\n \"endTime\": \"2026-09-16T10:08:52.4506315Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00153e0f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-9dwsgxpprw8c\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e39e5c6-0d40-4bf6-9634-0b867d3cf17c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e39e5c6-0d40-4bf6-9634-0b867d3cf17c\",\n \"name\": \"9e39e5c6-0d40-4bf6-9634-0b867d3cf17c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:44:48.885828252Z\",\n \"endTime\": \"2026-09-16T10:08:52.4506315Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40f7a558a9f14e3ad84a0dc62150fb9e1dcec198cbfa1c00cbbb104cbc63f3a7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":18,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","40f7a558a9f14e3ad84a0dc62150fb9e1dcec198cbfa1c00cbbb104cbc63f3a7","68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","a9f77fe91a65ae001825b7e195168037cff1cc1fd2f1b2a967813a5f71db56d2","f7e85048bd6764cc5537fb876507c8173ed1c8737be07b8c67fd9fcfdae58024"],"matched_occurrences":18,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132115432607744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6964,"pr_state":"closed","pr_sha":"af707ce72df7d034eaee96beafbefdfe37fc0800","final_merged_sha":"af707ce72df7d034eaee96beafbefdfe37fc0800","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T07:58:09Z","started_at":"2026-09-16T07:58:09Z","completed_at":"2026-09-16T10:11:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:58.491503217Z","timing_metadata_checked_at":"2026-09-18T15:40:58.491503217Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:58.494592728Z","region_metadata_checked_at":"2026-09-18T15:40:58.494592728Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"4a383b36bf0b1ad644cf8bafb119a2cb4f77bc39b67fa3cfbc2eb4d74efc997a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132115432607744","occurred_at":"2026-09-16T07:58:09Z","failed_at":"e2e","signature_id":"76ed85811ec3a7dce6c999043c0cc9d5ea6c96b382b975abd6e4372f70524c64","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00045b6a0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-tbrwcb86jfr2\" to finish creating: GET https://rp.j2607744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/100f602b-1abd-4d84-a8d9-1fc862b0b35f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/100f602b-1abd-4d84-a8d9-1fc862b0b35f\",\n \"name\": \"100f602b-1abd-4d84-a8d9-1fc862b0b35f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T08:49:15.881620207Z\",\n \"endTime\": \"2026-09-16T09:18:21.907934243Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00045b6a0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-tbrwcb86jfr2\" to finish creating: GET https://rp.j2607744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/100f602b-1abd-4d84-a8d9-1fc862b0b35f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/100f602b-1abd-4d84-a8d9-1fc862b0b35f\",\n \"name\": \"100f602b-1abd-4d84-a8d9-1fc862b0b35f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T08:49:15.881620207Z\",\n \"endTime\": \"2026-09-16T09:18:21.907934243Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9fd4af0f7c88edb25b900b9f12955f76ddd65cdbf946dcd98c71166362659422","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132115432607744","occurred_at":"2026-09-16T07:58:09Z","failed_at":"alert","signature_id":"2202a1a6bca53c40fa9b99a6151f7edd9b592782635ef9dfab434b81cf067278","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T09:09:50Z\n Ended: 2026-09-16T09:39:52Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2607744-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2607744/providers/microsoft.monitor/accounts/hcps-j2607744\", namespace=\"ocm-arohcpci01-2ss0k398kmvs03u4ijn3q0ehd6e40uln-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-nwfmp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2ss0k398kmvs03u4ijn3q0ehd6e40uln-agg-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132093936799744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6952,"pr_state":"closed","pr_sha":"a4de05dadb773673a7d1a63ee2677dbb182bda10","final_merged_sha":"a4de05dadb773673a7d1a63ee2677dbb182bda10","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-16T07:58:04Z","started_at":"2026-09-16T07:58:04Z","completed_at":"2026-09-16T10:14:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:56.933620914Z","timing_metadata_checked_at":"2026-09-18T15:40:56.933620914Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:56.936684525Z","region_metadata_checked_at":"2026-09-18T15:40:56.936684525Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132080049459200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6879,"pr_state":"closed","pr_sha":"9a437bfcf992a960d8de9f953fc1f640c6533fa8","final_merged_sha":"865e554719be3a2a33ad3c12d3fddff783af5356","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T07:58:01Z","started_at":"2026-09-16T07:58:01Z","completed_at":"2026-09-16T10:28:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:29.385187613Z","timing_metadata_checked_at":"2026-09-18T15:40:29.385187613Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:29.388181123Z","region_metadata_checked_at":"2026-09-18T15:40:29.388181123Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"2d6e60f7a595369d6d57d66da248d73ae955e91ebcd5b7f72768c4b4b87007d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132080049459200","occurred_at":"2026-09-16T07:58:01Z","failed_at":"e2e","signature_id":"1f776ef61b45ee091fed66db65a127929ae7ae7228c6ef8a3304e727c23cbe5d","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--w2v2r67444v5/cluster-hs-pre-7fdw2l should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e546a0\u003e: \n failed to create HCP cluster cluster-hs-pre-7fdw2l: failed waiting for cluster=\"cluster-hs-pre-7fdw2l\" in resourcegroup=\"rg-hs-pre--w2v2r67444v5\" to finish creating: GET https://rp.j9459200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1bcb8510-6d64-40ac-8f34-a62f6104e1e0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1bcb8510-6d64-40ac-8f34-a62f6104e1e0\",\n \"name\": \"1bcb8510-6d64-40ac-8f34-a62f6104e1e0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:04:50.385784561Z\",\n \"endTime\": \"2026-09-16T09:33:55.262197127Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 24m7s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--w2v2r67444v5/cluster-hs-pre-7fdw2l should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e546a0\u003e: \n failed to create HCP cluster cluster-hs-pre-7fdw2l: failed waiting for cluster=\"cluster-hs-pre-7fdw2l\" in resourcegroup=\"rg-hs-pre--w2v2r67444v5\" to finish creating: GET https://rp.j9459200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1bcb8510-6d64-40ac-8f34-a62f6104e1e0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1bcb8510-6d64-40ac-8f34-a62f6104e1e0\",\n \"name\": \"1bcb8510-6d64-40ac-8f34-a62f6104e1e0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:04:50.385784561Z\",\n \"endTime\": \"2026-09-16T09:33:55.262197127Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 24m7s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bb3d2a4be686b9827d4ee980fc11780fbf4e344eeca276a38d135903af33db04","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132080049459200","occurred_at":"2026-09-16T07:58:01Z","failed_at":"e2e","signature_id":"9f89db0ec4b8c3901dcf3ea1dec0354567233659a0ca459f22959ae94e89dbca","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001064760\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-fxln9tn9zv6z\" to finish creating: GET https://rp.j9459200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e3f68377-a68e-4f20-b0b6-eb88dfb8b7dc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e3f68377-a68e-4f20-b0b6-eb88dfb8b7dc\",\n \"name\": \"e3f68377-a68e-4f20-b0b6-eb88dfb8b7dc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:06:01.673328236Z\",\n \"endTime\": \"2026-09-16T09:35:05.314564461Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m56s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001064760\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-fxln9tn9zv6z\" to finish creating: GET https://rp.j9459200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e3f68377-a68e-4f20-b0b6-eb88dfb8b7dc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e3f68377-a68e-4f20-b0b6-eb88dfb8b7dc\",\n \"name\": \"e3f68377-a68e-4f20-b0b6-eb88dfb8b7dc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:06:01.673328236Z\",\n \"endTime\": \"2026-09-16T09:35:05.314564461Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m56s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cb6bc65dd341cc1e58e1b2b3508e6c2777582cb3ff74eb13579d434d75416bae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132080049459200","occurred_at":"2026-09-16T07:58:01Z","failed_at":"e2e","signature_id":"25f5f479c873e8672b346f86e1757b6d6cdd4c81dcdc1af82909656949ba3709","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b10860\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-7npmvs46gpv5\" to finish creating: GET https://rp.j9459200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e154370e-5d92-4a60-b860-409b26880120\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e154370e-5d92-4a60-b860-409b26880120\",\n \"name\": \"e154370e-5d92-4a60-b860-409b26880120\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:05:11.363311377Z\",\n \"endTime\": \"2026-09-16T09:34:15.227803795Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b10860\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-7npmvs46gpv5\" to finish creating: GET https://rp.j9459200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e154370e-5d92-4a60-b860-409b26880120\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e154370e-5d92-4a60-b860-409b26880120\",\n \"name\": \"e154370e-5d92-4a60-b860-409b26880120\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:05:11.363311377Z\",\n \"endTime\": \"2026-09-16T09:34:15.227803795Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cd477caf061571eb24882743948668a80aa53062da9a1e8e9258760e3f926918","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132080049459200","occurred_at":"2026-09-16T07:58:01Z","failed_at":"alert","signature_id":"6113c0f8fe30ddb0cec1586532a7684c6dc0ec614d444f96448deb0f4ad3a170","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T09:36:29Z\n Ended: 2026-09-16T09:42:31Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j9459200-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j9459200-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"da8170fa96505c8a9af9ec02dbc5e74e210439812b3ad6ec717b841298e80041","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132080049459200","occurred_at":"2026-09-16T07:58:01Z","failed_at":"alert","signature_id":"74727d2f774b690907a06926c56a326fbc39e84388995b5a3c7eceec1a64f678","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 6 time(s) (5 unknown, 1 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T09:15:02Z\n Ended: 2026-09-16T10:05:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9459200-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0r8p1f5df24425nn4ov1oc2b2s9qv-d2n3f2p5e8a4e8g\", pod=\"router-67f446dfd-cc8xg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0r8p1f5df24425nn4ov1oc2b2s9qv-d2n3f2p5e8a4e8g/router-67f446dfd-cc8xg has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-16T09:16:00Z\n Ended: 2026-09-16T09:44:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9459200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0rcsdsi36qo49ntkanduupf2g1ep1-q6m6u0v2u8u0a3w\", pod=\"router-6959bd5fcf-gghkx\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0rcsdsi36qo49ntkanduupf2g1ep1-q6m6u0v2u8u0a3w/router-6959bd5fcf-gghkx has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-16T09:17:01Z\n Ended: 2026-09-16T09:42:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9459200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0rid7on74rka1kfmd8b56931isqa8-w8i3f2p7v0w9p4u\", pod=\"router-66f89d764-wmnzq\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0rid7on74rka1kfmd8b56931isqa8-w8i3f2p7v0w9p4u/router-66f89d764-wmnzq has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-16T09:17:01Z\n Ended: 2026-09-16T09:31:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9459200-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0rfch8u2p7djo0cafmu1q2bpj2mq1-n2z8d3k4t3x1f7v\", pod=\"router-66bbd6fb7d-bkxxd\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0rfch8u2p7djo0cafmu1q2bpj2mq1-n2z8d3k4t3x1f7v/router-66bbd6fb7d-bkxxd has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-16T09:47:03Z\n Ended: 2026-09-16T09:53:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9459200-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss14p1akq7nrsvh7882pljokv1oijq0-h1u2z7i1y5s1m9c\", pod=\"openshift-apiserver-796668569c-jjbgg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss14p1akq7nrsvh7882pljokv1oijq0-h1u2z7i1y5s1m9c/openshift-apiserver-796668569c-jjbgg has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"ef3586484413088106cae20ade3c1b00b0449b298e599b842aaf481e40e4781a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132080049459200","occurred_at":"2026-09-16T07:58:01Z","failed_at":"e2e","signature_id":"817dbbd2f4e987cf783899d57ba9ab565857a252a91d7741c25fbfc506aff5e8","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132075855155200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6933,"pr_state":"closed","pr_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","final_merged_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T07:57:59Z","started_at":"2026-09-16T07:57:59Z","completed_at":"2026-09-16T10:36:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:42.793936753Z","timing_metadata_checked_at":"2026-09-18T15:40:42.793936753Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:42.797941967Z","region_metadata_checked_at":"2026-09-18T15:40:42.797941967Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"5a38270d885c3a1090bce495b4107418591723cb65caa4cc2789a32406109f4e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132075855155200","occurred_at":"2026-09-16T07:57:59Z","failed_at":"alert","signature_id":"4e60a01b4827c68ff57b07edc9eab558ce75aad5112478ce2737bee6696f871b","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T09:29:21Z\n Ended: 2026-09-16T09:49:20Z\n Severity: Sev3\n Labels: alertname=\"userJourneyFrontendLatencyP996h30m\", cluster=\"ci01-j5155200-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5155200/providers/microsoft.monitor/accounts/services-j5155200\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n Description: Frontend p99 latency on cluster ci01-j5155200-svc is above 1s on both the 6h and 30m windows (p99 \u003c 1s ARM sync SLO miss).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"52bfcbbc720001a7392b6a1e08f396579b61443952ed16c2c80c98fe6498c690","failure_pattern":"alert [svc] userJourneyFrontendLatencyP996h30m fired","search_query":"alert [svc] userJourneyFrontendLatencyP996h30m fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["52bfcbbc720001a7392b6a1e08f396579b61443952ed16c2c80c98fe6498c690"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132066686406656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6820,"pr_state":"open","pr_sha":"c99d95a578f89e3e4c3bfc476a031bb4b59056f4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T07:57:56Z","started_at":"2026-09-16T07:57:56Z","completed_at":"2026-09-16T09:20:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:25.175522968Z","timing_metadata_checked_at":"2026-09-18T15:40:25.175522968Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:25.179537882Z","region_metadata_checked_at":"2026-09-18T15:40:25.179537882Z"},"failed_at":["alert","provision"],"failed_test_count":4,"occurrences":[{"row_id":"392d60e97f679dfc73515eb735e8ee6ddbe92652a3d3a57faffd6f779cb9febc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132066686406656","occurred_at":"2026-09-16T07:57:56Z","failed_at":"provision","signature_id":"d08b0cbd15edc9cffd195b1bd96feff56d9f329451cf7393af1293efbc3c0c96","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","test_suite":"step graph","failure_text":"time=2026-09-16T08:43:27.720Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-16T08:43:28.143Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j6406656-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:94cee54eb85d08891b179bd74be48632bf6e7968f046225abb6074b4655891d0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:100m memory:256Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j6406656-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:0af23e7645c07968b9bd20f5c4a2966b34ccdaf8a05812b5eb5a3a26c53f24b5 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:510fcd49-73a6-4643-b104-d1da046c7a87 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#510fcd49-73a6-4643-b104-d1da046c7a87 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-16T08:43:28.143Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:43:28.283Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-16T08:43:28.434Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-16T08:43:28.434Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:43:28.493Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:53:31.202Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T08:53:31.203Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T08:54:31.204Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-16T08:54:31.661Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j6406656-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:94cee54eb85d08891b179bd74be48632bf6e7968f046225abb6074b4655891d0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:100m memory:256Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j6406656-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:0af23e7645c07968b9bd20f5c4a2966b34ccdaf8a05812b5eb5a3a26c53f24b5 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:510fcd49-73a6-4643-b104-d1da046c7a87 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#510fcd49-73a6-4643-b104-d1da046c7a87 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-16T08:54:31.661Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:54:31.919Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-16T08:54:32.413Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:54:32.413Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:54:32.686Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-16T08:54:33.497Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:54:33.498Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T08:54:33.902Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T08:54:34.404Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T08:54:34.757Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T08:54:34.953Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T08:54:35.182Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T08:54:35.544Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T08:54:36.068Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-16T08:54:36.068Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:54:36.176Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-16T09:04:37.727Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T09:04:37.727Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T09:05:37.733Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-16T09:05:38.373Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j6406656-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:94cee54eb85d08891b179bd74be48632bf6e7968f046225abb6074b4655891d0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:100m memory:256Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j6406656-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:0af23e7645c07968b9bd20f5c4a2966b34ccdaf8a05812b5eb5a3a26c53f24b5 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:510fcd49-73a6-4643-b104-d1da046c7a87 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#510fcd49-73a6-4643-b104-d1da046c7a87 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-16T09:05:38.373Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T09:05:38.676Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-16T09:05:39.023Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T09:05:39.023Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T09:05:39.181Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-16T09:05:40.018Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T09:05:40.018Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T09:05:40.264Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T09:05:40.639Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T09:05:41.058Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T09:05:41.352Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T09:05:41.683Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T09:05:42.063Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T09:05:42.276Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-16T09:05:42.276Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T09:05:42.359Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-16T09:13:27.645Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"context deadline exceeded\"\ntime=2026-09-16T09:13:27.646Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","failure_pattern":"failed to roll out Helm release: context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded"}},{"row_id":"6590050d8a04156b83f2ec7e6fc3333aeafa296512d46268290757bd9d47b0ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132066686406656","occurred_at":"2026-09-16T07:57:56Z","failed_at":"alert","signature_id":"358902433200cc0a70d0c371110e19da053abecdbf25175a02fa1df21c363226","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T08:59:42Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j6406656-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.98:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6406656/providers/microsoft.monitor/accounts/services-j6406656\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5979b8454d-h8gf2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-16T08:59:42Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j6406656-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.98:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6406656/providers/microsoft.monitor/accounts/services-j6406656\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5979b8454d-h8gf2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired"}},{"row_id":"6a3a068246d9a38671df8b7c7e23ed1b3c4e14ff5089fe085ee0aa0990a571d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132066686406656","occurred_at":"2026-09-16T07:57:56Z","failed_at":"alert","signature_id":"5c93d0c045993799007f8590467470292210fd23c8f5530b0425e5382165f354","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T08:49:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6406656-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-jpn9f\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-jpn9f has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-16T08:49:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6406656-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-cp72m\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-cp72m has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-16T08:49:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6406656-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-bcjjv\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-bcjjv has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-16T08:49:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6406656-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-z2nhx\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-z2nhx has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"9af46af041c737b5b6fc1efcb846f7c6ebf35278bf772de7ddd43aabf14bec09","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132066686406656","occurred_at":"2026-09-16T07:57:56Z","failed_at":"alert","signature_id":"cdf1b6098bdd9c6cf95b2228b3d492b2e30cb0b5e195c0384f719de68055f224","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T09:00:34Z\n Severity: Sev3\n Labels: alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j6406656-svc\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"warning\"\n Description: The Arobit forwarder metrics endpoint on cluster ci01-j6406656-svc has been unreachable for 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132038643290112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6503,"pr_state":"open","pr_sha":"6c5cfd09b9bbf1585b1cd37220da16677f327452","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T07:57:51Z","started_at":"2026-09-16T07:57:51Z","completed_at":"2026-09-16T10:41:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:08.927718407Z","timing_metadata_checked_at":"2026-09-18T15:40:08.927718407Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:08.930691417Z","region_metadata_checked_at":"2026-09-18T15:40:08.930691417Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"1a427272dec458412a60c86409eba596be297219b1e980dfcf4b0841ea4fb202","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132038643290112","occurred_at":"2026-09-16T07:57:51Z","failed_at":"alert","signature_id":"9daba40c60f37faf2af38abc3e3474d5917ef206c95bff1353f0bdaf3df9ac3b","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T09:46:29Z\n Severity: Sev3\n Labels: alertname=\"userJourneyFrontendLatencyP996h30m\", cluster=\"ci01-j3290112-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3290112/providers/microsoft.monitor/accounts/services-j3290112\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n Description: Frontend p99 latency on cluster ci01-j3290112-svc is above 1s on both the 6h and 30m windows (p99 \u003c 1s ARM sync SLO miss).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"52bfcbbc720001a7392b6a1e08f396579b61443952ed16c2c80c98fe6498c690","failure_pattern":"alert [svc] userJourneyFrontendLatencyP996h30m fired","search_query":"alert [svc] userJourneyFrontendLatencyP996h30m fired"}},{"row_id":"227f9d6c4928f94b581e70edc6f34072ca6a617eacf1c069b69b97995d3b5bc8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132038643290112","occurred_at":"2026-09-16T07:57:51Z","failed_at":"alert","signature_id":"bf9a145d432cea73856e5e5cd0fb2d2856d4a87e3d9fdb2343a2f93ed338bb52","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T09:25:35Z\n Ended: 2026-09-16T09:57:36Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3290112-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3290112-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"5742c871167a71da90e19e4ec618efa2b8657978849041aa0bf2df4d6c374cd0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132038643290112","occurred_at":"2026-09-16T07:57:51Z","failed_at":"e2e","signature_id":"fb548ce2c81d7ac7aefb5c63ff2cf1623addd485c6b89a7815a4071a2a0cf5c7","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.37\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010522c0\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-9vtzm4\" in resourcegroup=\"rg-np-noedge-9vtzm4-knfsxvjkqt2p\" to finish creating: GET https://rp.j3290112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1ae91cd5-981b-49a0-a69e-9b3ed247379a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1ae91cd5-981b-49a0-a69e-9b3ed247379a\",\n \"name\": \"1ae91cd5-981b-49a0-a69e-9b3ed247379a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:25:52.734930194Z\",\n \"endTime\": \"2026-09-16T09:26:31.102447106Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.37\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010522c0\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-9vtzm4\" in resourcegroup=\"rg-np-noedge-9vtzm4-knfsxvjkqt2p\" to finish creating: GET https://rp.j3290112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1ae91cd5-981b-49a0-a69e-9b3ed247379a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1ae91cd5-981b-49a0-a69e-9b3ed247379a\",\n \"name\": \"1ae91cd5-981b-49a0-a69e-9b3ed247379a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T09:25:52.734930194Z\",\n \"endTime\": \"2026-09-16T09:26:31.102447106Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"78189da791a9d4ba59876b732311c6105dd48c391d287147a80d1b8033d9a468","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132038643290112","occurred_at":"2026-09-16T07:57:51Z","failed_at":"alert","signature_id":"13f9b4b06b1b9850ff1976e03ab24843f0b045626fd218be7150c7dace6ffbef","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T09:39:29Z\n Ended: 2026-09-16T09:50:27Z\n Severity: Sev3\n Labels: alertname=\"userJourneyFrontendLatencyP991h5m\", cluster=\"ci01-j3290112-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3290112/providers/microsoft.monitor/accounts/services-j3290112\", severity=\"3\", short_window=\"5m\", slo=\"frontend-latency\"\n Description: Frontend p99 latency on cluster ci01-j3290112-svc is above 5s on both the 1h and 5m windows (severe miss of the p99 \u003c 1s ARM sync SLO).\nFiring 2:\n State: Resolved\n Started: 2026-09-16T10:10:33Z\n Ended: 2026-09-16T10:17:29Z\n Severity: Sev3\n Labels: alertname=\"userJourneyFrontendLatencyP991h5m\", cluster=\"ci01-j3290112-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3290112/providers/microsoft.monitor/accounts/services-j3290112\", severity=\"3\", short_window=\"5m\", slo=\"frontend-latency\"\n Description: Frontend p99 latency on cluster ci01-j3290112-svc is above 5s on both the 1h and 5m windows (severe miss of the p99 \u003c 1s ARM sync SLO).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d89bec3ea75e80fed983a698bbef1fbb78c33e77e634e6400cab6b214e710fe5","failure_pattern":"alert [svc] userJourneyFrontendLatencyP991h5m fired","search_query":"alert [svc] userJourneyFrontendLatencyP991h5m fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["52bfcbbc720001a7392b6a1e08f396579b61443952ed16c2c80c98fe6498c690","a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","d89bec3ea75e80fed983a698bbef1fbb78c33e77e634e6400cab6b214e710fe5","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6962/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132023019507712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6962,"pr_state":"closed","pr_sha":"3a661fc46e0956656c6fa07f1bab801eb15a77eb","final_merged_sha":"3a661fc46e0956656c6fa07f1bab801eb15a77eb","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T07:57:47Z","started_at":"2026-09-16T07:57:47Z","completed_at":"2026-09-16T10:32:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:58.057304056Z","timing_metadata_checked_at":"2026-09-18T15:40:58.057304056Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:58.063499378Z","region_metadata_checked_at":"2026-09-18T15:40:58.063499378Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"1b9b57610bebf1ece6e5554e019b6d6d3519b368c16e4837932ec433ae0fbcb2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6962/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132023019507712","occurred_at":"2026-09-16T07:57:47Z","failed_at":"alert","signature_id":"22a7d0041fe9eca9fd2523ee31fb7da254fed5da06967c558019791f8f2e970a","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T09:15:02Z\n Ended: 2026-09-16T09:25:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9507712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0r1rv65slu402q5o4341pkordffhr-d7b4p7w0c7w3c0o\", pod=\"router-6cb9c55d7c-4j7ns\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0r1rv65slu402q5o4341pkordffhr-d7b4p7w0c7w3c0o/router-6cb9c55d7c-4j7ns has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-16T09:17:03Z\n Ended: 2026-09-16T09:29:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9507712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0r6qa08bae603bidfs62a1vfvaur8-c3q5v4q2r7f5t2m\", pod=\"router-8596496b74-m9vph\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0r6qa08bae603bidfs62a1vfvaur8-c3q5v4q2r7f5t2m/router-8596496b74-m9vph has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132012407918592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6639,"pr_state":"closed","pr_sha":"bf8010e057a19fde91319030c3cc0c3ad980df3a","final_merged_sha":"bf8010e057a19fde91319030c3cc0c3ad980df3a","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T07:57:44Z","started_at":"2026-09-16T07:57:44Z","completed_at":"2026-09-16T08:32:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:09.168011572Z","timing_metadata_checked_at":"2026-09-18T15:40:09.168011572Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:09.171331784Z","region_metadata_checked_at":"2026-09-18T15:40:09.171331784Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"08d93ad3665391c98d2a67dacffc77d1da817b7a54ebb3fc8fcb7532aca2eef0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132012407918592","occurred_at":"2026-09-16T07:57:44Z","failed_at":"provision","signature_id":"09b1458b46da16205cd209ec402488a728d06d1e258b36845618ba3741f96692","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/infra-output","test_suite":"step graph","failure_text":"time=2026-09-16T08:27:04.451Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output description=\"Step infra-output\\n Kind: ARM\\n Template: templates/svc-infra-lookup.bicep\\n Parameters: configurations/svc-infra-lookup.tmpl.bicepparam\"\ntime=2026-09-16T08:27:04.849Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output\ntime=2026-09-16T08:27:05.446Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output deployment=eaaad5ae26368062d8da5183fba1f1cf527c2665c3e53af22a660fa55c8dc157 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7918592-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Feaaad5ae26368062d8da5183fba1f1cf527c2665c3e53af22a660fa55c8dc157\ntime=2026-09-16T08:27:35.588Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7918592-svc/providers/Microsoft.Resources/deployments/eaaad5ae26368062d8da5183fba1f1cf527c2665c3e53af22a660fa55c8dc157/operationStatuses/08584120596603628303\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentActive\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/svc-kv-xehipxbjxawki' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '9/16/2026 8:27:06 AM' with correlationId '032c9251-9fc6-4ff2-b8e3-30d15533261b', and will expire at '9/23/2026 8:27:06 AM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0008473d183c6e76249427529630789e6d4f9e8f1d5bdbea4b6bd57ae83c8f06","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentActive; detail message The deployment with resource id '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Resources/deployments/svc-kv-xehipxbjxawki' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '\u003ctimestamp\u003e' with correlationId '\u003cuuid\u003e', and will expire at '\u003ctimestamp\u003e' if it does not complete before then.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0008473d183c6e76249427529630789e6d4f9e8f1d5bdbea4b6bd57ae83c8f06"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132003708932096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6009,"pr_state":"open","pr_sha":"e76a45ff5a18f42766cc8fe18fe5bc1ca6b38a6d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T07:57:42Z","started_at":"2026-09-16T07:57:42Z","completed_at":"2026-09-16T09:14:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:04.18204412Z","timing_metadata_checked_at":"2026-09-18T15:40:04.18204412Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:04.185221632Z","region_metadata_checked_at":"2026-09-18T15:40:04.185221632Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"3a8caa4dd9dc618b108a503ab2c1b9423c3733b03bdfa2163832e64c8d83bd33","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132003708932096","occurred_at":"2026-09-16T07:57:42Z","failed_at":"provision","signature_id":"4592cebb946834bf2f5597cc8f68b2f39cdbe688bce563e773bde0e8a997725a","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kubelet-ds (stamp=2)","test_suite":"step graph","failure_text":"time=2026-09-16T09:03:26.822Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 description=\"Step kubelet-ds\\n Kind: Helm\\n\"\ntime=2026-09-16T09:03:27.157Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 values=\"map[enabled:true kubeletParametersImage:mcr.microsoft.com/aks/command/runtime@sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23]\"\ntime=2026-09-16T09:03:27.157Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-09-16T09:03:27.284Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 namespace=kubelet-ds\ntime=2026-09-16T09:03:27.475Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 dryRun=false\ntime=2026-09-16T09:03:27.475Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-09-16T09:03:27.530Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-09-16T09:08:28.583Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 err=\"resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 0/18\\ncontext deadline exceeded\"\ntime=2026-09-16T09:08:28.583Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 0/18\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e45f8e323244cd43106961bb8120aab24b3a867987bf8529ce7b7b4509a092cc","failure_pattern":"failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 0/18 context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 0/18\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e45f8e323244cd43106961bb8120aab24b3a867987bf8529ce7b7b4509a092cc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100129038159843328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6672,"pr_state":"closed","pr_sha":"6ce8aee4f672e78e5466ddffb3d8139dd153568d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T07:45:55Z","started_at":"2026-09-16T07:45:55Z","completed_at":"2026-09-16T10:00:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:09.700964691Z","timing_metadata_checked_at":"2026-09-18T15:40:09.700964691Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:09.704104902Z","region_metadata_checked_at":"2026-09-18T15:40:09.704104902Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"24ec27903b8f872db11c4ec5ae6012e5351f0b8bf862e749f60cf5f8d2f26f55","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100129038159843328","occurred_at":"2026-09-16T07:45:55Z","failed_at":"alert","signature_id":"05b02974c8e5aa95fe3326e72aadd1d9c24fabbae417684be2c2a88ea3d78009","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 43 time(s) (31 unknown, 12 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T08:46:36Z\n Ended: 2026-09-16T09:11:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0e1d6e28cgcj8srsgji99eu63mo4i-a2r9l9g4p0t9a9l\", pod=\"kube-state-metrics-hcp-8654bffc85-ps27z\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0e1d6e28cgcj8srsgji99eu63mo4i-a2r9l9g4p0t9a9l/kube-state-metrics-hcp-8654bffc85-ps27z has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-16T08:46:36Z\n Ended: 2026-09-16T09:04:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0duf4s9stu0aprgf4ol4j3c99jk2f-j8v4h6s4u8u8p4l\", pod=\"kube-state-metrics-hcp-8654bffc85-cwpmw\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0duf4s9stu0aprgf4ol4j3c99jk2f-j8v4h6s4u8u8p4l/kube-state-metrics-hcp-8654bffc85-cwpmw has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-16T08:47:34Z\n Ended: 2026-09-16T09:06:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0dtmd7phkt983ei3vk9l5hqge338m-z6z2n3u3s4c1i5m\", pod=\"kube-state-metrics-hcp-8654bffc85-tvn6p\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0dtmd7phkt983ei3vk9l5hqge338m-z6z2n3u3s4c1i5m/kube-state-metrics-hcp-8654bffc85-tvn6p has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-16T08:48:34Z\n Ended: 2026-09-16T09:06:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0e9nto594s9bprvj5apj58rjljcjh-n0r0v2f3i9x3l1e\", pod=\"kube-state-metrics-hcp-8654bffc85-q4qrz\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0e9nto594s9bprvj5apj58rjljcjh-n0r0v2f3i9x3l1e/kube-state-metrics-hcp-8654bffc85-q4qrz has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-16T08:48:34Z\n Ended: 2026-09-16T09:11:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0e5616jettcj0e7uig00q85cf1bug-q7s6x3r1w1e1g4u\", pod=\"kube-state-metrics-hcp-8654bffc85-p8q8r\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0e5616jettcj0e7uig00q85cf1bug-q7s6x3r1w1e1g4u/kube-state-metrics-hcp-8654bffc85-p8q8r has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-16T08:48:34Z\n Ended: 2026-09-16T09:03:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0drohbr45u43inc6u449jd9sevca4-sre-hcp-cluster\", pod=\"kube-state-metrics-hcp-8654bffc85-sclrb\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0drohbr45u43inc6u449jd9sevca4-sre-hcp-cluster/kube-state-metrics-hcp-8654bffc85-sclrb has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-16T08:48:34Z\n Ended: 2026-09-16T09:11:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0ds27bmfd0lp6hvok9h387svmutrh-m7o7e1m4q4g5a0q\", pod=\"kube-state-metrics-hcp-8654bffc85-kw5xd\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0ds27bmfd0lp6hvok9h387svmutrh-m7o7e1m4q4g5a0q/kube-state-metrics-hcp-8654bffc85-kw5xd has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-16T08:48:34Z\n Ended: 2026-09-16T09:26:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0e5482l6g3gjjck8dmgk09unp5n62-w4i9v7z8p2f3l7h\", pod=\"kube-state-metrics-hcp-8654bffc85-c2lnm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0e5482l6g3gjjck8dmgk09unp5n62-w4i9v7z8p2f3l7h/kube-state-metrics-hcp-8654bffc85-c2lnm has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-16T08:49:36Z\n Ended: 2026-09-16T09:11:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0e0se573hope116jkdbf3q1bsdf8b-cilium-cl\", pod=\"kube-state-metrics-hcp-8654bffc85-xvxb9\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0e0se573hope116jkdbf3q1bsdf8b-cilium-cl/kube-state-metrics-hcp-8654bffc85-xvxb9 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-16T08:49:36Z\n Ended: 2026-09-16T09:02:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0dulm91k97f02qma0s0l25hkbd2na-basic-cluster\", pod=\"kube-state-metrics-hcp-8654bffc85-nc62p\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0dulm91k97f02qma0s0l25hkbd2na-basic-cluster/kube-state-metrics-hcp-8654bffc85-nc62p has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-16T08:49:36Z\n Ended: 2026-09-16T09:22:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0e299ccde1v5r1t1qpok7ges0ddr0-c7r4k0l0o7y7f7g\", pod=\"kube-state-metrics-hcp-8654bffc85-884m9\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0e299ccde1v5r1t1qpok7ges0ddr0-c7r4k0l0o7y7f7g/kube-state-metrics-hcp-8654bffc85-884m9 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-16T08:49:36Z\n Ended: 2026-09-16T09:23:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0e2d6ioi48inqqd9othg3996mh958-k1z1b8a3k4u4i9i\", pod=\"kube-state-metrics-hcp-8654bffc85-cm49w\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0e2d6ioi48inqqd9othg3996mh958-k1z1b8a3k4u4i9i/kube-state-metrics-hcp-8654bffc85-cm49w has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-09-16T08:50:35Z\n Ended: 2026-09-16T09:06:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0dscenj77pmhb82oqe4hmj9hquq7f-a4i8a2i9w1m2q8f\", pod=\"kube-state-metrics-hcp-8654bffc85-8426r\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0dscenj77pmhb82oqe4hmj9hquq7f-a4i8a2i9w1m2q8f/kube-state-metrics-hcp-8654bffc85-8426r has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-09-16T08:50:35Z\n Ended: 2026-09-16T09:11:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0e1lj58vfiilrpgklc8aev2a8vhm2-y3c2u9u6p8i8f1j\", pod=\"kube-state-metrics-hcp-8654bffc85-q85rl\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0e1lj58vfiilrpgklc8aev2a8vhm2-y3c2u9u6p8i8f1j/kube-state-metrics-hcp-8654bffc85-q85rl has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-09-16T08:50:35Z\n Ended: 2026-09-16T09:21:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0e2brb3h1abmdb9q6t7s3jhggu4pa-t3s9n8v8m7h8n4b\", pod=\"kube-state-metrics-hcp-8654bffc85-kqjcf\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0e2brb3h1abmdb9q6t7s3jhggu4pa-t3s9n8v8m7h8n4b/kube-state-metrics-hcp-8654bffc85-kqjcf has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-09-16T08:52:35Z\n Ended: 2026-09-16T09:11:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0dtrihd5vifelghabc0c05hniti06-w6n8j9u9k7n7b1g\", pod=\"kube-state-metrics-hcp-8654bffc85-9njg2\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0dtrihd5vifelghabc0c05hniti06-w6n8j9u9k7n7b1g/kube-state-metrics-hcp-8654bffc85-9njg2 has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-09-16T08:52:35Z\n Ended: 2026-09-16T09:09:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0e57qf4hvm7piged8okdoteu9ep64-c3r3v6g0k6b2t0y\", pod=\"kube-state-metrics-hcp-8654bffc85-t5t9l\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0e57qf4hvm7piged8okdoteu9ep64-c3r3v6g0k6b2t0y/kube-state-metrics-hcp-8654bffc85-t5t9l has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-09-16T08:52:35Z\n Ended: 2026-09-16T09:11:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0dtbtkug6eg8cvsj1ip575q4h6g8k-q6g6h8g9t3b7p2f\", pod=\"kube-state-metrics-hcp-8654bffc85-786vd\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0dtbtkug6eg8cvsj1ip575q4h6g8k-q6g6h8g9t3b7p2f/kube-state-metrics-hcp-8654bffc85-786vd has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-09-16T08:52:35Z\n Ended: 2026-09-16T09:20:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0dsa50hk3db67ji91qm6ggb7bgd5g-r1x4c0m1g1y2p3q\", pod=\"kube-state-metrics-hcp-8654bffc85-xj7w4\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0dsa50hk3db67ji91qm6ggb7bgd5g-r1x4c0m1g1y2p3q/kube-state-metrics-hcp-8654bffc85-xj7w4 has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-09-16T08:52:35Z\n Ended: 2026-09-16T09:13:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0dto8jouf46i1bb19k20eh7vjukqa-ea-cluster\", pod=\"kube-state-metrics-hcp-8654bffc85-cz9jw\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0dto8jouf46i1bb19k20eh7vjukqa-ea-cluster/kube-state-metrics-hcp-8654bffc85-cz9jw has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-09-16T08:52:35Z\n Ended: 2026-09-16T09:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0e0gpe5eqkn8mba3h9lv428t445t4-s3v5h2p8x5u6m6o\", pod=\"kube-state-metrics-hcp-8654bffc85-tv58x\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0e0gpe5eqkn8mba3h9lv428t445t4-s3v5h2p8x5u6m6o/kube-state-metrics-hcp-8654bffc85-tv58x has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-09-16T08:52:35Z\n Ended: 2026-09-16T09:13:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0e3ndrc2bqqgvpe0os0ru0qu81d4p-f5r7a5r7u7s6n9a\", pod=\"kube-state-metrics-hcp-8654bffc85-qzm78\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0e3ndrc2bqqgvpe0os0ru0qu81d4p-f5r7a5r7u7s6n9a/kube-state-metrics-hcp-8654bffc85-qzm78 has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-09-16T08:52:35Z\n Ended: 2026-09-16T09:09:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0e5qo1hiar598l8vimo23riu0p3og-ea-listp6l6j7\", pod=\"kube-state-metrics-hcp-8654bffc85-ffpsk\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0e5qo1hiar598l8vimo23riu0p3og-ea-listp6l6j7/kube-state-metrics-hcp-8654bffc85-ffpsk has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-09-16T08:52:35Z\n Ended: 2026-09-16T09:34:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0e5e2tf5cdj027ehc6v6447aemoft-l6c5i1r6w9j9a1g\", pod=\"kube-state-metrics-hcp-8654bffc85-g42k6\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0e5e2tf5cdj027ehc6v6447aemoft-l6c5i1r6w9j9a1g/kube-state-metrics-hcp-8654bffc85-g42k6 has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-09-16T08:52:35Z\n Ended: 2026-09-16T09:11:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0e1g1dpcqdb9vlq6fnutb7ef82jou-n9x3t3q0v5w7k4b\", pod=\"kube-state-metrics-hcp-8654bffc85-ml4d9\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0e1g1dpcqdb9vlq6fnutb7ef82jou-n9x3t3q0v5w7k4b/kube-state-metrics-hcp-8654bffc85-ml4d9 has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-09-16T08:52:35Z\n Ended: 2026-09-16T09:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0eh4306o4cvqj7ckcj3sgvioip60o-n1l9j4i7v8l0i3d\", pod=\"kube-state-metrics-hcp-8654bffc85-wf9nr\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0eh4306o4cvqj7ckcj3sgvioip60o-n1l9j4i7v8l0i3d/kube-state-metrics-hcp-8654bffc85-wf9nr has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-09-16T08:52:35Z\n Ended: 2026-09-16T09:06:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0e9gihrbuhf99vlkgkmat5gh1gj57-x9h0l9t2j6n2x9h\", pod=\"kube-state-metrics-hcp-8654bffc85-s4594\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0e9gihrbuhf99vlkgkmat5gh1gj57-x9h0l9t2j6n2x9h/kube-state-metrics-hcp-8654bffc85-s4594 has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-09-16T08:52:35Z\n Ended: 2026-09-16T09:20:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0e1ink1tm5udcqm74e4ndq08g2ria-g4t9b4q5g1f3v9e\", pod=\"kube-state-metrics-hcp-8654bffc85-kdd4f\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0e1ink1tm5udcqm74e4ndq08g2ria-g4t9b4q5g1f3v9e/kube-state-metrics-hcp-8654bffc85-kdd4f has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-09-16T08:52:35Z\n Ended: 2026-09-16T09:41:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0e2i0dvn6r1kqn5krgm1e1odtoe99-f6s4u2c4f9l9e2a\", pod=\"kube-state-metrics-hcp-8654bffc85-5pxfx\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0e2i0dvn6r1kqn5krgm1e1odtoe99-f6s4u2c4f9l9e2a/kube-state-metrics-hcp-8654bffc85-5pxfx has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-09-16T08:52:35Z\n Ended: 2026-09-16T09:20:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0e5afk4l93i8hn8lrot867h8r4cm4-a3r0x6m0c1h1t2j\", pod=\"kube-state-metrics-hcp-8654bffc85-f2577\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0e5afk4l93i8hn8lrot867h8r4cm4-a3r0x6m0c1h1t2j/kube-state-metrics-hcp-8654bffc85-f2577 has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Resolved\n Started: 2026-09-16T08:52:35Z\n Ended: 2026-09-16T09:13:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9843328-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss0du7tfltom0c1644hm0cgcokcqo6b-n3k9i3w8r8w9h2o\", pod=\"kube-state-metrics-hcp-8654bffc85-2w4nd\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss0du7tfltom0c1644hm0cgcokcqo6b-n3k9i3w8r8w9h2o/kube-state-metrics-hcp-8654bffc85-2w4nd has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"52e49e434324b37a3b68220315ddd7fdf36db471194c2a4cd0c94b8ea72dad8c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100129038159843328","occurred_at":"2026-09-16T07:45:55Z","failed_at":"alert","signature_id":"89db9db5751f9f51402e00b217747d3b3ea1c62487d699b8eeaa41599ef274e2","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T09:13:39Z\n Ended: 2026-09-16T09:19:34Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9843328-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-state-metrics-hcp\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9843328/providers/microsoft.monitor/accounts/hcps-j9843328\", namespace=\"ocm-arohcpci01-2ss0e2d6ioi48inqqd9othg3996mh958-k1z1b8a3k4u4i9i\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-jlv8v\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ss0e2d6ioi48inqqd9othg3996mh958-k1z1b8a3k4u4i9i/kube-state-metrics-hcp has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-16T09:13:39Z\n Ended: 2026-09-16T09:19:34Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9843328-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-state-metrics-hcp\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9843328/providers/microsoft.monitor/accounts/hcps-j9843328\", namespace=\"ocm-arohcpci01-2ss0e299ccde1v5r1t1qpok7ges0ddr0-c7r4k0l0o7y7f7g\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-jlv8v\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ss0e299ccde1v5r1t1qpok7ges0ddr0-c7r4k0l0o7y7f7g/kube-state-metrics-hcp has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-16T09:13:39Z\n Ended: 2026-09-16T09:27:38Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9843328-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-state-metrics-hcp\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9843328/providers/microsoft.monitor/accounts/hcps-j9843328\", namespace=\"ocm-arohcpci01-2ss0e5482l6g3gjjck8dmgk09unp5n62-w4i9v7z8p2f3l7h\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-jlv8v\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ss0e5482l6g3gjjck8dmgk09unp5n62-w4i9v7z8p2f3l7h/kube-state-metrics-hcp has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"8eb459a2f0258b962ae58f61177c36f29a64e5d783d46289bcea1c6304f09c06","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100129038159843328","occurred_at":"2026-09-16T07:45:55Z","failed_at":"e2e","signature_id":"e60b8a3f0f846e4caf38925534828b81ff9b59a90b098e189f25e9987eb8d737","test_name":"AMA Metrics service metrics from SVC and MGMT clusters should be present in Azure Monitor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/ama_metrics.go:83]: failed to look up SVC Prometheus endpoint for workspace services-yt in resource group hcp-underlay-ci01-yt\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001004ba0\u003e: \n failed to get workspace services-yt: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-yt/providers/Microsoft.Monitor/accounts/services-yt\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceGroupNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-yt' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/ama_metrics.go:83]: failed to look up SVC Prometheus endpoint for workspace services-yt in resource group hcp-underlay-ci01-yt\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001004ba0\u003e: \n failed to get workspace services-yt: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-yt/providers/Microsoft.Monitor/accounts/services-yt\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceGroupNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-yt' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"575b0de6701cb1cdcdd3b5c4e6523248471d3aa67c3c0c1fed243233e4014b33","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.Monitor","search_query":"ERROR CODE: ResourceGroupNotFound"}},{"row_id":"d5c71893f853f38e041cb0cc69f732e0b36dabe63d6bbb8d19489ddc0f1c5220","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100129038159843328","occurred_at":"2026-09-16T07:45:55Z","failed_at":"alert","signature_id":"cbb7931aab2d0ed1916b8d84363dfe1ffc05cf35d9ce851f977710db9cdf561f","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T08:53:50Z\n Ended: 2026-09-16T09:06:52Z\n Severity: Sev3\n Labels: alertname=\"userJourneyFrontendLatencyP996h30m\", cluster=\"ci01-j9843328-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9843328/providers/microsoft.monitor/accounts/services-j9843328\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n Description: Frontend p99 latency on cluster ci01-j9843328-svc is above 1s on both the 6h and 30m windows (p99 \u003c 1s ARM sync SLO miss).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"52bfcbbc720001a7392b6a1e08f396579b61443952ed16c2c80c98fe6498c690","failure_pattern":"alert [svc] userJourneyFrontendLatencyP996h30m fired","search_query":"alert [svc] userJourneyFrontendLatencyP996h30m fired"}},{"row_id":"e13a0ffe190448153767d89715fb913653df20b55629614ddcdb3b57b32a4acf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100129038159843328","occurred_at":"2026-09-16T07:45:55Z","failed_at":"e2e","signature_id":"4a840acfa7ba40488e61bba537bd235d758cbe0b71e71c1cdf5af1d8876188b2","test_name":"KSM HCP Metrics kube_node_info metrics should be present in Azure Monitor for the happy-path cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:74]: failed to look up HCP Prometheus endpoint\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c94e20\u003e: \n failed to get workspace hcps-yt: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-yt/providers/Microsoft.Monitor/accounts/hcps-yt\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceGroupNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-yt' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:74]: failed to look up HCP Prometheus endpoint\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c94e20\u003e: \n failed to get workspace hcps-yt: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-yt/providers/Microsoft.Monitor/accounts/hcps-yt\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceGroupNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-yt' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"575b0de6701cb1cdcdd3b5c4e6523248471d3aa67c3c0c1fed243233e4014b33","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.Monitor","search_query":"ERROR CODE: ResourceGroupNotFound"}},{"row_id":"ec1648ec8f21af6117331351c3f91a284c9c15ace8454c589097042cd4cf3c68","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100129038159843328","occurred_at":"2026-09-16T07:45:55Z","failed_at":"e2e","signature_id":"12729704797975e796e396a1c7d5eb085f228e272a9762c5cf68851df1b74ece","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.003s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000a540c0\u003e: \n failed to get kusto cluster name from config: configuration: key kusto not found\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.003s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000a540c0\u003e: \n failed to get kusto cluster name from config: configuration: key kusto not found\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c71ebc2a93d9fc10a75dcaf39a2b1325de6379c3891238b64d1e4e8f4281df9a","failure_pattern":"failed to get kusto cluster \u003ckusto-cluster\u003e from config: configuration: key kusto not found","search_query":"failed to get kusto cluster name from config: configuration: key kusto not found"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","52bfcbbc720001a7392b6a1e08f396579b61443952ed16c2c80c98fe6498c690","575b0de6701cb1cdcdd3b5c4e6523248471d3aa67c3c0c1fed243233e4014b33","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","c71ebc2a93d9fc10a75dcaf39a2b1325de6379c3891238b64d1e4e8f4281df9a"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6955/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100127664214904832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6955,"pr_state":"closed","pr_sha":"3772fbb2e2b95267ceee977b6fa31cb974604a39","final_merged_sha":"3772fbb2e2b95267ceee977b6fa31cb974604a39","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-16T07:40:28Z","started_at":"2026-09-16T07:40:28Z","completed_at":"2026-09-16T10:05:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:57.900225291Z","timing_metadata_checked_at":"2026-09-18T15:40:57.900225291Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:57.904440406Z","region_metadata_checked_at":"2026-09-18T15:40:57.904440406Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6971/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100127403039789056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6971,"pr_state":"closed","pr_sha":"ea9e0ef11ac9b22da4762ee1b3c145bb7eee1a05","final_merged_sha":"ea9e0ef11ac9b22da4762ee1b3c145bb7eee1a05","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T07:39:25Z","started_at":"2026-09-16T07:39:25Z","completed_at":"2026-09-16T09:54:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:02.083134641Z","timing_metadata_checked_at":"2026-09-18T15:41:02.083134641Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:02.086493454Z","region_metadata_checked_at":"2026-09-18T15:41:02.086493454Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"56b38d20c12d7ac24c6797918f6877c2403aa972861cfd011e4c2ef398eabbec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6971/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100127403039789056","occurred_at":"2026-09-16T07:39:25Z","failed_at":"alert","signature_id":"02815353fa0a66ddf20b88ae30c92eac6b08249d84094735fa13cf0c3952861f","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T08:58:25Z\n Ended: 2026-09-16T09:33:27Z\n Severity: Sev3\n Labels: alertname=\"userJourneyFrontendLatencyP996h30m\", cluster=\"ci01-j9789056-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9789056/providers/microsoft.monitor/accounts/services-j9789056\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n Description: Frontend p99 latency on cluster ci01-j9789056-svc is above 1s on both the 6h and 30m windows (p99 \u003c 1s ARM sync SLO miss).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"52bfcbbc720001a7392b6a1e08f396579b61443952ed16c2c80c98fe6498c690","failure_pattern":"alert [svc] userJourneyFrontendLatencyP996h30m fired","search_query":"alert [svc] userJourneyFrontendLatencyP996h30m fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["52bfcbbc720001a7392b6a1e08f396579b61443952ed16c2c80c98fe6498c690"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100127266641022976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6791,"pr_state":"closed","pr_sha":"764ee60fa32d166f92c6c286f96007787ee189de","final_merged_sha":"00c5d478babb26a92f08e45ee9f4c322bafd7507","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-16T07:38:53Z","started_at":"2026-09-16T07:38:53Z","completed_at":"2026-09-16T09:52:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:22.603090313Z","timing_metadata_checked_at":"2026-09-18T15:40:22.603090313Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:22.607611529Z","region_metadata_checked_at":"2026-09-18T15:40:22.607611529Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100124639266082816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6856,"pr_state":"closed","pr_sha":"bc69b5f8bdc86dcc19d05452e41ea7526512b8dd","final_merged_sha":"96f38fd5a277adea0aeec323ef5685124af19024","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T07:28:27Z","started_at":"2026-09-16T07:28:27Z","completed_at":"2026-09-16T09:41:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:25.798382109Z","timing_metadata_checked_at":"2026-09-18T15:40:25.798382109Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:25.80143952Z","region_metadata_checked_at":"2026-09-18T15:40:25.80143952Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"66d6598d6cffcef4df492a35cc37a925fc39c725f47a99f1879bfde22ea61760","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100124639266082816","occurred_at":"2026-09-16T07:28:27Z","failed_at":"alert","signature_id":"1bb0a319d54a133be2e86c848056c08984c613c97a376359e9e2876ee27aa821","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T08:27:37Z\n Ended: 2026-09-16T08:40:39Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6082816-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ss03n4euqvgrodqiodqlsm6e44lqi3v-p9n3x2f1x8p9s9a\", pod=\"router-7589fd99c7-22p68\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ss03n4euqvgrodqiodqlsm6e44lqi3v-p9n3x2f1x8p9s9a/router-7589fd99c7-22p68 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100119365197238272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6939,"pr_state":"closed","pr_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","final_merged_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T07:07:29Z","started_at":"2026-09-16T07:07:29Z","completed_at":"2026-09-16T09:23:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:44.440116378Z","timing_metadata_checked_at":"2026-09-18T15:40:44.440116378Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:44.443716791Z","region_metadata_checked_at":"2026-09-18T15:40:44.443716791Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"29d8ca5d8aad64a8d9a6c62b70b7711cf998a5b4ff673807b02b8b2abc9cc5f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100119365197238272","occurred_at":"2026-09-16T07:07:29Z","failed_at":"e2e","signature_id":"f6b4ca993a39a3f8a63f6cce0ebaa94328998c8c08a83ab0da84bb108e61207b","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c6ad00\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-xpjdxspbvkfj\" to finish creating: GET https://rp.j7238272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4d55f5f9-f6e8-4345-b2b0-d96a01eab89e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4d55f5f9-f6e8-4345-b2b0-d96a01eab89e\",\n \"name\": \"4d55f5f9-f6e8-4345-b2b0-d96a01eab89e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T07:56:30.963864533Z\",\n \"endTime\": \"2026-09-16T08:25:39.96844469Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m2s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c6ad00\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-xpjdxspbvkfj\" to finish creating: GET https://rp.j7238272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4d55f5f9-f6e8-4345-b2b0-d96a01eab89e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4d55f5f9-f6e8-4345-b2b0-d96a01eab89e\",\n \"name\": \"4d55f5f9-f6e8-4345-b2b0-d96a01eab89e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T07:56:30.963864533Z\",\n \"endTime\": \"2026-09-16T08:25:39.96844469Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m2s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ce2feb889d2bdb258ade0aa09b20431dc56148354e1dd35d7c41beced9ea7782","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100119365197238272","occurred_at":"2026-09-16T07:07:29Z","failed_at":"e2e","signature_id":"8485b82a0c9f334d47923eba74cfc4cebce6914810909ffc70f3acbe8579b2bf","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010c81a0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-xbxmwmzgbdf6\" to finish creating: GET https://rp.j7238272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b2412533-f3f2-47d5-9634-4c30b5400a83\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b2412533-f3f2-47d5-9634-4c30b5400a83\",\n \"name\": \"b2412533-f3f2-47d5-9634-4c30b5400a83\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T07:55:54.64211789Z\",\n \"endTime\": \"2026-09-16T08:24:59.944024206Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m37s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010c81a0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-xbxmwmzgbdf6\" to finish creating: GET https://rp.j7238272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b2412533-f3f2-47d5-9634-4c30b5400a83\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b2412533-f3f2-47d5-9634-4c30b5400a83\",\n \"name\": \"b2412533-f3f2-47d5-9634-4c30b5400a83\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T07:55:54.64211789Z\",\n \"endTime\": \"2026-09-16T08:24:59.944024206Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m37s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100119215724826624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6972,"pr_state":"closed","pr_sha":"299202de6ed4a011da948403847bb418b1ce0fbf","final_merged_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T07:06:53Z","started_at":"2026-09-16T07:06:53Z","completed_at":"2026-09-16T08:11:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:02.383350523Z","timing_metadata_checked_at":"2026-09-18T15:41:02.383350523Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:02.386612634Z","region_metadata_checked_at":"2026-09-18T15:41:02.386612634Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"56c1e01b55b6da994a4bd28d13ae9c450e1687041f5f35f1737ebfc6fd80a0a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100119215724826624","occurred_at":"2026-09-16T07:06:53Z","failed_at":"provision","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","test_suite":"step graph","failure_text":"time=2026-09-16T07:35:17.577Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-16T07:35:18.216Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-09-16T07:35:19.015Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=d9196c0050ee21da58d027182daf82fc87c36acf0a933ff3ded4321f5c85c815 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4826624-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fd9196c0050ee21da58d027182daf82fc87c36acf0a933ff3ded4321f5c85c815\ntime=2026-09-16T08:05:17.276Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100115702714208256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5724,"pr_state":"closed","pr_sha":"5e814b754c6ac49bdda39af9047f424419d1397c","final_merged_sha":"02f55855ad2e600a742ce1b551002c7fbd872350","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-16T06:52:56Z","started_at":"2026-09-16T06:52:56Z","completed_at":"2026-09-16T09:19:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:02.454901408Z","timing_metadata_checked_at":"2026-09-18T15:40:02.454901408Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:02.458022919Z","region_metadata_checked_at":"2026-09-18T15:40:02.458022919Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100115652357394432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"open","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-16T06:52:44Z","started_at":"2026-09-16T06:52:44Z","completed_at":"2026-09-16T09:05:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:54.088001179Z","timing_metadata_checked_at":"2026-09-18T15:40:54.088001179Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:54.091370291Z","region_metadata_checked_at":"2026-09-18T15:40:54.091370291Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100112882132848640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6953,"pr_state":"closed","pr_sha":"b8f732681cc510ab7e2b385dd50715f0140fe81a","final_merged_sha":"b8f732681cc510ab7e2b385dd50715f0140fe81a","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-16T06:41:43Z","started_at":"2026-09-16T06:41:43Z","completed_at":"2026-09-16T08:55:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:22.756926122Z","timing_metadata_checked_at":"2026-09-18T15:41:22.756926122Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:22.759803133Z","region_metadata_checked_at":"2026-09-18T15:41:22.759803133Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100110536946487296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6880,"pr_state":"closed","pr_sha":"5d038e66cb652d84493a82df7e161483f83459f0","final_merged_sha":"a20dd280dd6b7499cf853051219487314ef1e460","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-16T06:32:24Z","started_at":"2026-09-16T06:32:24Z","completed_at":"2026-09-16T08:52:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:30.556719826Z","timing_metadata_checked_at":"2026-09-18T15:40:30.556719826Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:30.560269939Z","region_metadata_checked_at":"2026-09-18T15:40:30.560269939Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6923/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100110113200148480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6923,"pr_state":"closed","pr_sha":"9e6650394c80bca242363572272841073766ea22","final_merged_sha":"9e6650394c80bca242363572272841073766ea22","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T06:30:43Z","started_at":"2026-09-16T06:30:43Z","completed_at":"2026-09-16T08:53:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:41.146157122Z","timing_metadata_checked_at":"2026-09-18T15:40:41.146157122Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:41.149616635Z","region_metadata_checked_at":"2026-09-18T15:40:41.149616635Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"098dc2e60a16575bdc90b7077e9fd5186b84097b87d255a782b741bbc7d12bc6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6923/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100110113200148480","occurred_at":"2026-09-16T06:30:43Z","failed_at":"alert","signature_id":"d87f770fb784724db359656072889a1f3015ce39f593db987908535455d1ee42","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T08:04:47Z\n Ended: 2026-09-16T08:25:46Z\n Severity: Sev3\n Labels: alertname=\"userJourneyFrontendLatencyP996h30m\", cluster=\"ci01-j0148480-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0148480/providers/microsoft.monitor/accounts/services-j0148480\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n Description: Frontend p99 latency on cluster ci01-j0148480-svc is above 1s on both the 6h and 30m windows (p99 \u003c 1s ARM sync SLO miss).\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-16T08:41:46Z\n Severity: Sev3\n Labels: alertname=\"userJourneyFrontendLatencyP996h30m\", cluster=\"ci01-j0148480-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0148480/providers/microsoft.monitor/accounts/services-j0148480\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n Description: Frontend p99 latency on cluster ci01-j0148480-svc is above 1s on both the 6h and 30m windows (p99 \u003c 1s ARM sync SLO miss).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"52bfcbbc720001a7392b6a1e08f396579b61443952ed16c2c80c98fe6498c690","failure_pattern":"alert [svc] userJourneyFrontendLatencyP996h30m fired","search_query":"alert [svc] userJourneyFrontendLatencyP996h30m fired"}},{"row_id":"a02485607a50d6b9015e8151330ea6911ada500f4ebece58ad6c548d540f9325","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6923/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100110113200148480","occurred_at":"2026-09-16T06:30:43Z","failed_at":"alert","signature_id":"fac8088074452240e37582cff853c6a9cc2e040277fbf34b005056861933699a","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T07:44:18Z\n Ended: 2026-09-16T07:50:17Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j0148480-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} has exceeded 250ms over the past 30 minutes (cluster ci01-j0148480-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["52bfcbbc720001a7392b6a1e08f396579b61443952ed16c2c80c98fe6498c690","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100109263216054272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6923,"pr_state":"closed","pr_sha":"9e6650394c80bca242363572272841073766ea22","final_merged_sha":"9e6650394c80bca242363572272841073766ea22","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-16T06:27:10Z","started_at":"2026-09-16T06:27:10Z","completed_at":"2026-09-16T08:47:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:22.57399496Z","timing_metadata_checked_at":"2026-09-18T15:41:22.57399496Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:22.577384572Z","region_metadata_checked_at":"2026-09-18T15:41:22.577384572Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100104517457547264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6894,"pr_state":"closed","pr_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","final_merged_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T06:08:29Z","started_at":"2026-09-16T06:08:29Z","completed_at":"2026-09-16T08:38:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:33.108992005Z","timing_metadata_checked_at":"2026-09-18T15:40:33.108992005Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:33.112971019Z","region_metadata_checked_at":"2026-09-18T15:40:33.112971019Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"3cc820df696ccccec1c08924bc24e7cd7739bc16563523df334fd6930057ae0c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100104517457547264","occurred_at":"2026-09-16T06:08:29Z","failed_at":"alert","signature_id":"5f54bcb248196b637bd98df597f0841d329c7e41ad40465254245b7458522f8c","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-16T07:38:34Z\n Ended: 2026-09-16T07:45:33Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7547264-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7547264-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"f241c69ef2e3db4440b6715ffcd671303657e1f5efce19b85e0f8b2c4f2e05e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100104517457547264","occurred_at":"2026-09-16T06:08:29Z","failed_at":"alert","signature_id":"d21b60b6e4bcd8f18d5041389daab6c5b24d976173fd091a6cd2e7e5a098f762","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-16T07:52:13Z\n Severity: Sev3\n Labels: alertname=\"userJourneyFrontendLatencyP996h30m\", cluster=\"ci01-j7547264-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7547264/providers/microsoft.monitor/accounts/services-j7547264\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n Description: Frontend p99 latency on cluster ci01-j7547264-svc is above 1s on both the 6h and 30m windows (p99 \u003c 1s ARM sync SLO miss).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"52bfcbbc720001a7392b6a1e08f396579b61443952ed16c2c80c98fe6498c690","failure_pattern":"alert [svc] userJourneyFrontendLatencyP996h30m fired","search_query":"alert [svc] userJourneyFrontendLatencyP996h30m fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["52bfcbbc720001a7392b6a1e08f396579b61443952ed16c2c80c98fe6498c690","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100079940652240896","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6917,"pr_state":"open","pr_sha":"55efd56d7b76c88f48195a55ad9df67aaf1f0f91","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T04:30:50Z","timing_metadata_state":"forbidden","timing_metadata_first_checked_at":"2026-09-18T15:42:14.97207444Z","timing_metadata_checked_at":"2026-09-18T15:42:14.97207444Z","region_metadata_state":"forbidden","region_metadata_first_checked_at":"2026-09-18T15:42:14.975283851Z","region_metadata_checked_at":"2026-09-18T15:42:14.975283851Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"8acf5a43b8a4f1b9896089f28e5001c46ec9446024e6676de129db0932e842fc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100079940652240896","occurred_at":"2026-09-16T04:30:50Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100076986327109632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6964,"pr_state":"closed","pr_sha":"af707ce72df7d034eaee96beafbefdfe37fc0800","final_merged_sha":"af707ce72df7d034eaee96beafbefdfe37fc0800","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T04:19:05Z","timing_metadata_state":"forbidden","timing_metadata_first_checked_at":"2026-09-18T15:42:32.447106692Z","timing_metadata_checked_at":"2026-09-18T15:42:32.447106692Z","region_metadata_state":"forbidden","region_metadata_first_checked_at":"2026-09-18T15:42:32.450099602Z","region_metadata_checked_at":"2026-09-18T15:42:32.450099602Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"83b2bd393e4680677fc0a25312ecea7374ecad864f179e86481caa060ead9b2b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100076986327109632","occurred_at":"2026-09-16T04:19:05Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100067480113254400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6547,"pr_state":"closed","pr_sha":"84cdee396e856d537c5404f7af4e8d0f95216389","final_merged_sha":"84cdee396e856d537c5404f7af4e8d0f95216389","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T03:41:19Z","timing_metadata_state":"forbidden","timing_metadata_first_checked_at":"2026-09-18T15:41:42.86940467Z","timing_metadata_checked_at":"2026-09-18T15:41:42.86940467Z","region_metadata_state":"forbidden","region_metadata_first_checked_at":"2026-09-18T15:41:42.872543882Z","region_metadata_checked_at":"2026-09-18T15:41:42.872543882Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"9edfabd8f7288738f5d59af5bd1c68741e472c050790cdd02bd21c290306ef9e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100067480113254400","occurred_at":"2026-09-16T03:41:19Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100066597707190272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6547,"pr_state":"closed","pr_sha":"84cdee396e856d537c5404f7af4e8d0f95216389","final_merged_sha":"84cdee396e856d537c5404f7af4e8d0f95216389","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-16T03:37:48Z","timing_metadata_state":"forbidden","timing_metadata_first_checked_at":"2026-09-18T15:42:40.582246227Z","timing_metadata_checked_at":"2026-09-18T15:42:40.582246227Z","region_metadata_state":"forbidden","region_metadata_first_checked_at":"2026-09-18T15:42:40.585380938Z","region_metadata_checked_at":"2026-09-18T15:42:40.585380938Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100060856170582016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6970,"pr_state":"open","pr_sha":"467befb45920afb78e18656f8b828865f8e7a829","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T03:14:59Z","timing_metadata_state":"forbidden","timing_metadata_first_checked_at":"2026-09-18T15:42:33.258106607Z","timing_metadata_checked_at":"2026-09-18T15:42:33.258106607Z","region_metadata_state":"forbidden","region_metadata_first_checked_at":"2026-09-18T15:42:33.262372422Z","region_metadata_checked_at":"2026-09-18T15:42:33.262372422Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"4eef21a86555fff47cae6506b87080e6e6bb2713b1acd64b30b28bf38d5f1af3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100060856170582016","occurred_at":"2026-09-16T03:14:59Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100044634842468352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5846,"pr_state":"open","pr_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T02:10:32Z","timing_metadata_state":"forbidden","timing_metadata_first_checked_at":"2026-09-18T15:42:40.49394761Z","timing_metadata_checked_at":"2026-09-18T15:42:40.49394761Z","region_metadata_state":"forbidden","region_metadata_first_checked_at":"2026-09-18T15:42:40.49685422Z","region_metadata_checked_at":"2026-09-18T15:42:40.49685422Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"4874c0e342db39a4f6b08d41da26293d214f9b001950ee1d95dea9f3c179cb5b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100044634842468352","occurred_at":"2026-09-16T02:10:32Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100040823386148864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"2d1acfb7fc671055def32e5be0bc50812a04245f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T01:55:23Z","timing_metadata_state":"forbidden","timing_metadata_first_checked_at":"2026-09-18T15:41:41.333923427Z","timing_metadata_checked_at":"2026-09-18T15:41:41.333923427Z","region_metadata_state":"forbidden","region_metadata_first_checked_at":"2026-09-18T15:41:41.337146639Z","region_metadata_checked_at":"2026-09-18T15:41:41.337146639Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"a1252802c0a1e7169fc408387bac04939f17a41d39799325c6052ce2262c106e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100040823386148864","occurred_at":"2026-09-16T01:55:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6961/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100036806081777664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6961,"pr_state":"closed","pr_sha":"7c53ac85c06ab4fabc8fce3d43e1ac08f3e1784e","final_merged_sha":"7c53ac85c06ab4fabc8fce3d43e1ac08f3e1784e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-16T01:39:25Z","timing_metadata_state":"forbidden","timing_metadata_first_checked_at":"2026-09-18T15:42:30.892204002Z","timing_metadata_checked_at":"2026-09-18T15:42:30.892204002Z","region_metadata_state":"forbidden","region_metadata_first_checked_at":"2026-09-18T15:42:30.895157713Z","region_metadata_checked_at":"2026-09-18T15:42:30.895157713Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100026257558212608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6966,"pr_state":"closed","pr_sha":"4856ca84dbf040eebde6a2cfd3b2a1d6f03b4ac4","final_merged_sha":"c93ff943c35ad2d21e3a98df5a27db691c9749a7","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-16T00:57:31Z","timing_metadata_state":"forbidden","timing_metadata_first_checked_at":"2026-09-18T15:42:32.918695087Z","timing_metadata_checked_at":"2026-09-18T15:42:32.918695087Z","region_metadata_state":"forbidden","region_metadata_first_checked_at":"2026-09-18T15:42:32.923978106Z","region_metadata_checked_at":"2026-09-18T15:42:32.923978106Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100026244044165120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6503,"pr_state":"open","pr_sha":"6c5cfd09b9bbf1585b1cd37220da16677f327452","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T00:57:27Z","timing_metadata_state":"forbidden","timing_metadata_first_checked_at":"2026-09-18T15:41:42.382330012Z","timing_metadata_checked_at":"2026-09-18T15:41:42.382330012Z","region_metadata_state":"forbidden","region_metadata_first_checked_at":"2026-09-18T15:41:42.385765624Z","region_metadata_checked_at":"2026-09-18T15:41:42.385765624Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ccb18fb7c76c9846e7d6953dea1353126886de028ace70d764260e1979062335","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100026244044165120","occurred_at":"2026-09-16T00:57:27Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100014987492724736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5846,"pr_state":"open","pr_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-16T00:12:39Z","timing_metadata_state":"forbidden","timing_metadata_first_checked_at":"2026-09-18T15:42:40.338327051Z","timing_metadata_checked_at":"2026-09-18T15:42:40.338327051Z","region_metadata_state":"forbidden","region_metadata_first_checked_at":"2026-09-18T15:42:40.341531562Z","region_metadata_checked_at":"2026-09-18T15:42:40.341531562Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"a9c8edea31b7ce3cadeb283782317061d99f8867341ab722dd5c7286fea2578f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100014987492724736","occurred_at":"2026-09-16T00:12:39Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100013933942280192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6894,"pr_state":"closed","pr_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","final_merged_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-16T00:01:03Z","timing_metadata_state":"forbidden","timing_metadata_first_checked_at":"2026-09-18T15:42:10.058682246Z","timing_metadata_checked_at":"2026-09-18T15:42:10.058682246Z","region_metadata_state":"forbidden","region_metadata_first_checked_at":"2026-09-18T15:42:10.061792457Z","region_metadata_checked_at":"2026-09-18T15:42:10.061792457Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]},{"environment":"int","summary":{"total_runs":4,"failed_runs":1,"runs_with_occurrences":1,"runs_with_failure_pattern_matches":1,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":3,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100351631508901888","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T22:30:19Z","started_at":"2026-09-16T22:30:19Z","completed_at":"2026-09-17T00:40:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:41.992442193Z","timing_metadata_checked_at":"2026-09-18T15:42:41.992442193Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:41.995843106Z","region_metadata_checked_at":"2026-09-18T15:42:41.995843106Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"13f3d7d57eacd4168bb4b1c0e3e02061a51fc4e2de00c22d4d8620ac983db1bb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100351631508901888","occurred_at":"2026-09-16T22:30:19Z","failed_at":"e2e","signature_id":"f7210bc017800c8d3ce9292e2b300557e56751eedb6338b78091f529330d0435","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:204]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc000df6cd8\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-pfttlc9b26s6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff088bfb-6289-4eb8-a3c7-2bd5dc2e8821\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff088bfb-6289-4eb8-a3c7-2bd5dc2e8821\",\n \"name\": \"ff088bfb-6289-4eb8-a3c7-2bd5dc2e8821\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T22:57:32.075098968Z\",\n \"endTime\": \"2026-09-16T23:16:32.974740274Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:204]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc000df6cd8\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-pfttlc9b26s6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff088bfb-6289-4eb8-a3c7-2bd5dc2e8821\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff088bfb-6289-4eb8-a3c7-2bd5dc2e8821\",\n \"name\": \"ff088bfb-6289-4eb8-a3c7-2bd5dc2e8821\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T22:57:32.075098968Z\",\n \"endTime\": \"2026-09-16T23:16:32.974740274Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a52d3f63256add31d93e084d23258696e3ce9f667510ec06dc509046b07f2f14","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a52d3f63256add31d93e084d23258696e3ce9f667510ec06dc509046b07f2f14"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100265227302998016","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-16T16:47:05Z","started_at":"2026-09-16T16:47:05Z","completed_at":"2026-09-16T18:59:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:41.843157957Z","timing_metadata_checked_at":"2026-09-18T15:42:41.843157957Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:41.846476069Z","region_metadata_checked_at":"2026-09-18T15:42:41.846476069Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100206685254586368","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-16T12:54:28Z","started_at":"2026-09-16T12:54:28Z","completed_at":"2026-09-16T15:06:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:41.681464376Z","timing_metadata_checked_at":"2026-09-18T15:42:41.681464376Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:41.687403097Z","region_metadata_checked_at":"2026-09-18T15:42:41.687403097Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100170707966103552","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-16T10:31:30Z","started_at":"2026-09-16T10:31:30Z","completed_at":"2026-09-16T12:44:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:41.550836607Z","timing_metadata_checked_at":"2026-09-18T15:42:41.550836607Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:41.556215426Z","region_metadata_checked_at":"2026-09-18T15:42:41.556215426Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]},{"environment":"prod","summary":{"total_runs":5,"failed_runs":5,"runs_with_occurrences":5,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":3,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T23:00:16Z","started_at":"2026-09-16T23:00:16Z","completed_at":"2026-09-17T01:05:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:48.81993973Z","timing_metadata_checked_at":"2026-09-18T15:42:48.81993973Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:48.823592043Z","region_metadata_checked_at":"2026-09-18T15:42:48.823592043Z"},"failed_at":["e2e"],"failed_test_count":8,"occurrences":[{"row_id":"2bbcbccbb467804481aa7847c64c57127a3298d733b81305e0fddd91d6f2c761","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","failed_at":"e2e","signature_id":"494d690c10401d515d7d97bb1b43282cd5512bedf4a8cf0e93991a312d0f71e1","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001d82db0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-d2hkj9sg5shk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001d82db0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-d2hkj9sg5shk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"4ea38501428aaf262df4412ed17c04925e6d3b01cca73ac6e4fe67b5db5c47d2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","failed_at":"e2e","signature_id":"70af6c4c5a716b4a2eb8db0539e648de82b2496a872a99a8c6a98074fd41d4c3","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d85560\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-jzkmnxzs9c52\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-jzkmnxzs9c52/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d85560\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-jzkmnxzs9c52\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-jzkmnxzs9c52/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d6164cff5eccd3db4fabd19fbe08aa12f39c6b3d5c2202182c9f291029d16282","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.22\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"573f0aefdc2d5379a8a61c9723efbf98b20ccda7192c46375294d5f52f8b8cc2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","failed_at":"e2e","signature_id":"5f0d2310ed06414a49e9f1ddd637c2a2516af067a00a8168124b28d160b1f9b0","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000fd73e0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-dqvpfm666hzz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000fd73e0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-dqvpfm666hzz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"65c48a28829a2bfee332a19df0d0f809af6c5728a8a2ea0c386731828191e6a2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","failed_at":"e2e","signature_id":"e2f2d3555780f9a4be9e4c49fe522f13754292efbc3c1299543a5305777d3351","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00133e180\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-pgw6tmx6nx48/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00133e180\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-pgw6tmx6nx48/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"6d462c439e9c2c47ae94225eda2ac705f38e363b7d3f4fac3fde0b8dd4013fa4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","failed_at":"e2e","signature_id":"a91c26d043927cb825f419af86a21ed7c54daa9e6da09ecdbe0fab7af0a8ab85","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002c97050\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-8h5dwhv96zkm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002c97050\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-8h5dwhv96zkm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"9acdf59e359bdb10f74013bb9452dabef296f95e2ceedcbd440c4cb7445cbd23","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","failed_at":"e2e","signature_id":"aae2457e0f58577b233ddb5159361da1324cc74ad01ffe504591fd6f9131c0dd","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3cd80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-b7c4p2nvk6d6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0259344c-f395-4714-abc8-4abafaaf707d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0259344c-f395-4714-abc8-4abafaaf707d\",\n \"name\": \"0259344c-f395-4714-abc8-4abafaaf707d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T23:18:34.031156052Z\",\n \"endTime\": \"2026-09-16T23:37:37.814154784Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3cd80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-b7c4p2nvk6d6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0259344c-f395-4714-abc8-4abafaaf707d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0259344c-f395-4714-abc8-4abafaaf707d\",\n \"name\": \"0259344c-f395-4714-abc8-4abafaaf707d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T23:18:34.031156052Z\",\n \"endTime\": \"2026-09-16T23:37:37.814154784Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c3272939304c23f824f7469f9128bff7596590c7bf09ca5e3caa4d71b56deb63","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","failed_at":"e2e","signature_id":"b2ee8d4e7b429ad69336ec70ed0fafeb7a58d728ccf645b31d75121fd78def37","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:274]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00139eea0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-q5hhjjfw9zhm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:274]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00139eea0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-q5hhjjfw9zhm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"df20252b3bc3c4e03719c349462d78d0a6ccd2882c76a22b3c6d25c37639cf85","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","failed_at":"e2e","signature_id":"f5ce501d2dd69dcdea5a25282314ce7ff4dede1a06ceeb918b7e36499f9665b6","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00152ab10\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00152ab10\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","d6164cff5eccd3db4fabd19fbe08aa12f39c6b3d5c2202182c9f291029d16282"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100173746357669888","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T10:43:35Z","started_at":"2026-09-16T10:43:35Z","completed_at":"2026-09-16T13:11:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:47.281062899Z","timing_metadata_checked_at":"2026-09-18T15:42:47.281062899Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:47.28425201Z","region_metadata_checked_at":"2026-09-18T15:42:47.28425201Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"abd1300941ca34fa14118a274792ce9f58b11598a44f1eeb7722a14239502cc5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100173746357669888","occurred_at":"2026-09-16T10:43:35Z","failed_at":"e2e","signature_id":"cfd34eb004cc5a7cf7577f79d571b368c1d8dfc53041aa652f9465c361a3b4a0","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:251]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n \u003c*errors.joinError | 0xc002a4e018\u003e: \n context deadline exceeded\n cluster operators not available: console (no Available condition), dns (Available=False, Progressing=True, Degraded=True, Upgradeable=True), image-registry (Available=False, Progressing=True, Degraded=True), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False), monitoring (no Available condition), openshift-samples (no Available condition), storage (Degraded=False, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown)\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:251]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n \u003c*errors.joinError | 0xc002a4e018\u003e: \n context deadline exceeded\n cluster operators not available: console (no Available condition), dns (Available=False, Progressing=True, Degraded=True, Upgradeable=True), image-registry (Available=False, Progressing=True, Degraded=True), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False), monitoring (no Available condition), openshift-samples (no Available condition), storage (Degraded=False, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown)\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b57f3261aa83aadd9ceb7988b00830acc29894d12dc4f5f19baa862276967565","failure_pattern":"cluster operators not available: console (no Available condition), dns (Available=False, Progressing=True, Degraded=True, Upgradeable=True), image-registry (Available=False, Progressing=True, Degraded=True), ingress (Available=False, Progressing=True","search_query":"cluster operators not available: console (no Available condition), dns (Available=False, Progressing=True, Degraded=True, Upgradeable=True), image-registry (Available=False, Progressing=True, Degraded=True), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False), monitoring (no Available condition), openshift-samples (no Available condition), storage (Degraded=False, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b57f3261aa83aadd9ceb7988b00830acc29894d12dc4f5f19baa862276967565"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100090888859422720","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T05:14:20Z","timing_metadata_state":"forbidden","timing_metadata_first_checked_at":"2026-09-18T15:42:50.930559217Z","timing_metadata_checked_at":"2026-09-18T15:42:50.930559217Z","region_metadata_state":"forbidden","region_metadata_first_checked_at":"2026-09-18T15:42:50.933847729Z","region_metadata_checked_at":"2026-09-18T15:42:50.933847729Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b1d2ae99cc5820f234babd7753c3aa93486d08bf47a4185c514abbd44f39fa22","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100090888859422720","occurred_at":"2026-09-16T05:14:20Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100042048844337152","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T02:00:15Z","timing_metadata_state":"forbidden","timing_metadata_first_checked_at":"2026-09-18T15:42:53.023805941Z","timing_metadata_checked_at":"2026-09-18T15:42:53.023805941Z","region_metadata_state":"forbidden","region_metadata_first_checked_at":"2026-09-18T15:42:53.026893352Z","region_metadata_checked_at":"2026-09-18T15:42:53.026893352Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"6ead95fa01885821c6b1fe50435188e7625bae866121583c005ecfd8cc440ea3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100042048844337152","occurred_at":"2026-09-16T02:00:15Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100020039049547776","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T00:32:48Z","timing_metadata_state":"forbidden","timing_metadata_first_checked_at":"2026-09-18T15:42:50.832081763Z","timing_metadata_checked_at":"2026-09-18T15:42:50.832081763Z","region_metadata_state":"forbidden","region_metadata_first_checked_at":"2026-09-18T15:42:50.835315975Z","region_metadata_checked_at":"2026-09-18T15:42:50.835315975Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"9ca3507498c744b65752e5f70fef33be2e0edabfc42cbea4623e16f0df7e5a08","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100020039049547776","occurred_at":"2026-09-16T00:32:48Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}}]},{"environment":"stg","summary":{"total_runs":2,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":1,"runs_with_unmatched_occurrences":1,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T23:00:16Z","started_at":"2026-09-16T23:00:16Z","completed_at":"2026-09-17T00:39:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:55.807626339Z","timing_metadata_checked_at":"2026-09-18T15:42:55.807626339Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:55.81076735Z","region_metadata_checked_at":"2026-09-18T15:42:55.81076735Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"1dd369f1d13d534f59a82dd6c9ea4aba73171e9dc2a5b9e31ef369e6087e10c8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","failed_at":"e2e","signature_id":"77617baa5bf45330c8ac3ac9717b66793245c58ebcc17c8a3e357cf43d6f0bf5","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0016e5a88\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0016e5a88\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"2367d5232576057fbaeaff488a2b1c4e1dfb3fc4412a4170b537922edcaaae82","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","failed_at":"e2e","signature_id":"8be57f60e7cd91c124c72d6d01098c6e1b8b853355d51b79f6b1bd55c240efdb","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0027deed0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-wck4dz5gg7vn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0027deed0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-wck4dz5gg7vn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"3b6f5aedabe791ef4797612bd19b3c4c90ffc83d096b9ea7432099a93ab75a91","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","failed_at":"e2e","signature_id":"51c0a7fe19269071bf4a86bc72b032196ba0e6e4e142e092ab1f3aca5f282e06","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-pbnkzl\" for cluster \"cluster-nightly-4-20-pbnkzl\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002c1a920\u003e: \n failed to create NodePool np-pbnkzl: failed starting nodepool creation \"np-pbnkzl\" for cluster \"cluster-nightly-4-20-pbnkzl\" in resourcegroup=\"rg-nightly-4-20-6c9ns9nbgrzm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-6c9ns9nbgrzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-pbnkzl/nodePools/np-pbnkzl\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-pbnkzl\" for cluster \"cluster-nightly-4-20-pbnkzl\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002c1a920\u003e: \n failed to create NodePool np-pbnkzl: failed starting nodepool creation \"np-pbnkzl\" for cluster \"cluster-nightly-4-20-pbnkzl\" in resourcegroup=\"rg-nightly-4-20-6c9ns9nbgrzm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-6c9ns9nbgrzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-pbnkzl/nodePools/np-pbnkzl\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"3c15a4d673ae1dd06fc1e16418f50831aedda9d6a2b2a19568d0e9ec40e6f5d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","failed_at":"e2e","signature_id":"a3bcc0cf85a56ec12c072c420e0931c744e3a86ca315e7af1b9dd0b787fa5b91","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-15-135250\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000facdc0\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-m58t8j\" resourcegroup=\"rg-np-dg-minor-m58t8j-trxh29mwfdlb\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-m58t8j-trxh29mwfdlb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-m58t8j/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-15-135250\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000facdc0\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-m58t8j\" resourcegroup=\"rg-np-dg-minor-m58t8j-trxh29mwfdlb\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-m58t8j-trxh29mwfdlb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-m58t8j/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"51e19d38c821c9cea8c398e1f440c72fa8ce08291870dfca5abd004310dc73ad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","failed_at":"e2e","signature_id":"3799930be8939ca48534a28b262ff33328138426e7cec1b9fb2793e6f4a95899","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001d4bc80\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-56f2xp2w67w8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001d4bc80\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-56f2xp2w67w8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"66a782654443936994843623b633b0c595c0f64416d713e78700153993fb18b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","failed_at":"e2e","signature_id":"18ab22781017136dd055b840d502183c7edf1caa41f67dcadf9fd0a7857c8081","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-krkxx8\" for cluster \"cluster-nightly-4-23-krkxx8\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00121ee40\u003e: \n failed to create NodePool np-krkxx8: failed waiting for nodepool=\"np-krkxx8\" for cluster \"cluster-nightly-4-23-krkxx8\" in resourcegroup=\"rg-nightly-4-23-7l9rdmzm7f2z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e59a6ba-7f62-48bf-8e29-7e6ddefd08f9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e59a6ba-7f62-48bf-8e29-7e6ddefd08f9\",\n \"name\": \"4e59a6ba-7f62-48bf-8e29-7e6ddefd08f9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T23:37:25.694607859Z\",\n \"endTime\": \"2026-09-16T23:56:27.737971571Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-krkxx8\" for cluster \"cluster-nightly-4-23-krkxx8\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00121ee40\u003e: \n failed to create NodePool np-krkxx8: failed waiting for nodepool=\"np-krkxx8\" for cluster \"cluster-nightly-4-23-krkxx8\" in resourcegroup=\"rg-nightly-4-23-7l9rdmzm7f2z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e59a6ba-7f62-48bf-8e29-7e6ddefd08f9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e59a6ba-7f62-48bf-8e29-7e6ddefd08f9\",\n \"name\": \"4e59a6ba-7f62-48bf-8e29-7e6ddefd08f9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-16T23:37:25.694607859Z\",\n \"endTime\": \"2026-09-16T23:56:27.737971571Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d5a27e23f87f5c18e786f32d10682561abfa5723896eb83d080ad6377beec74f","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"72883da5fbf64d5c58d71e1c712695d8979fa7bb7f28edbd7f77edf1ab4b38ff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","failed_at":"e2e","signature_id":"3d8d555b219d0ed8c9661c956fba32429ade34240dfb89932e7b6f838ec71e56","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-15-135250\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004d4640\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-c8fd7f\" in resourcegroup=\"rg-np-version-upgrade-c8fd7f-kdnr5znbw57r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-c8fd7f-kdnr5znbw57r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-c8fd7f/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-15-135250\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004d4640\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-c8fd7f\" in resourcegroup=\"rg-np-version-upgrade-c8fd7f-kdnr5znbw57r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-c8fd7f-kdnr5znbw57r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-c8fd7f/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"88bf7f5d526ab53e006ea71299716986fe5b80c3e12045ab0cb1766135fb8036","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","failed_at":"e2e","signature_id":"bd4c07c42b25870c0f1baaecfa45c3c668afd2f7347cf758e572515c16e421fd","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:274]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001682660\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-wjvk4n2mqnjj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:274]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001682660\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-wjvk4n2mqnjj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a5927166d82c10f31a395fa862a79cca5508a016e2e0b6637d50eddf7333ec49","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","failed_at":"e2e","signature_id":"c259628d57d4675c02d0130322edaa375c878722b676fb3290dbafba51543aa4","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-15-135250\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00063d580\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-w9gvcr\" in resourcegroup=\"rg-np-skip-minor-w9gvcr-bshw78xg4hsg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-w9gvcr-bshw78xg4hsg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-w9gvcr/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-15-135250\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00063d580\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-w9gvcr\" in resourcegroup=\"rg-np-skip-minor-w9gvcr-bshw78xg4hsg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-w9gvcr-bshw78xg4hsg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-w9gvcr/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"abb1a9ec62cb5425df67630a0b4a675ba235c8357c240eb40cfb3ae0c2d2668b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","failed_at":"e2e","signature_id":"7a42cc1723b1b8bf9843f1aa12dd8baec4448f6917d7f084b3a4b154baa63afa","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000f2fd40\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-g8sp2mm6ks67/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000f2fd40\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-g8sp2mm6ks67/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"de46e7510a7d4015471786027ded1d38f92b02412848a74c385dab0e026f68ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","failed_at":"e2e","signature_id":"5c2b3fa28b82ccf14825b20d7135a7cb3d0c7f8bdbad597c1b9eba4c000e0333","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010b5a80\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-2wnkdzqz6d9w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-2wnkdzqz6d9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010b5a80\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-2wnkdzqz6d9w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-2wnkdzqz6d9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5c1e65a74237eb97d61c283de75bcde39935ce10b1226f1bc067c35315e03a3d","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.22\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"fdbaf05abc99f32a70620d3d8b64706223f8f705259f0e725b5f539f048273fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","failed_at":"e2e","signature_id":"474ef928e4bfaa163ff38e770c6435689354cdc8821b9a05ec4272d8ad60c351","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0012fd950\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-57279mxs72dc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0012fd950\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-57279mxs72dc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","5c1e65a74237eb97d61c283de75bcde39935ce10b1226f1bc067c35315e03a3d","646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","d5a27e23f87f5c18e786f32d10682561abfa5723896eb83d080ad6377beec74f"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2100042048882085888","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-16T02:00:15Z","timing_metadata_state":"forbidden","timing_metadata_first_checked_at":"2026-09-18T15:43:01.220629579Z","timing_metadata_checked_at":"2026-09-18T15:43:01.220629579Z","region_metadata_state":"forbidden","region_metadata_first_checked_at":"2026-09-18T15:43:01.223846191Z","region_metadata_checked_at":"2026-09-18T15:43:01.223846191Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"a34fa30fad79ab2e1d2249ebcdeaf4e4fcac07f005a07c72d48b9d8b8eb319af","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2100042048882085888","occurred_at":"2026-09-16T02:00:15Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}}]}]}