{"meta":{"date":"2026-09-30","timezone":"UTC","generated_at":"2026-10-31T08:32:20Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":98,"failed_runs":52,"runs_with_occurrences":52,"runs_with_failure_pattern_matches":48,"runs_with_unmatched_occurrences":4,"runs_without_occurrence_rows":46,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7283/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105443667987664896","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7283,"pr_state":"closed","pr_sha":"e7158a2130aaa1ec2b66af7bdf7f99e356001c8e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T23:44:22Z","started_at":"2026-09-30T23:44:22Z","completed_at":"2026-10-01T01:52:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T01:59:03.291818604Z","timing_metadata_checked_at":"2026-10-01T01:59:03.291818604Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T01:59:03.295375593Z","region_metadata_checked_at":"2026-10-01T01:59:03.295375593Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"a34097ee209422fc4aa983050d49f5e9e7dae90eb79f1f4ddd5f58ce566da701","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7283/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105443667987664896","occurred_at":"2026-09-30T23:44:22Z","failed_at":"alert","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-01T00:40:12Z\n  Ended:       2026-10-01T00:54:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7664896-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7126/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105435642618974208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7126,"pr_state":"closed","pr_sha":"429533a4edff625c1d9b3f963d1f627dd55bfb9e","final_merged_sha":"54589bc135b17d9071a54f907162af4e9fa3ece9","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T23:12:28Z","started_at":"2026-09-30T23:12:28Z","completed_at":"2026-10-01T01:29:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T01:38:39.982578282Z","timing_metadata_checked_at":"2026-10-01T01:38:39.982578282Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T01:38:39.985671473Z","region_metadata_checked_at":"2026-10-01T01:38:39.985671473Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7258/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105434099047993344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7258,"pr_state":"closed","pr_sha":"d3ebe31be82921b12d03aac77751ad5e1513c442","final_merged_sha":"a68022814b830b9a8334e00ff48929e4987b9c4a","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T23:06:20Z","started_at":"2026-09-30T23:06:20Z","completed_at":"2026-10-01T01:34:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T01:38:40.33068001Z","timing_metadata_checked_at":"2026-10-01T01:38:40.33068001Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T01:38:40.333467201Z","region_metadata_checked_at":"2026-10-01T01:38:40.333467201Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7281/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105433677730156544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7281,"pr_state":"closed","pr_sha":"0f52db4929ddeb744ff754924b061ebef3af2dc2","final_merged_sha":"128ad3152db580e14c5bdced12edb9abbf3f9bab","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T23:04:40Z","started_at":"2026-09-30T23:04:40Z","completed_at":"2026-10-01T01:25:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T01:28:26.463798129Z","timing_metadata_checked_at":"2026-10-01T01:28:26.463798129Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T01:28:26.46690662Z","region_metadata_checked_at":"2026-10-01T01:28:26.46690662Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105433504048222208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7261,"pr_state":"closed","pr_sha":"f6667070546b139bd53a7df7d7674fd0c4250636","final_merged_sha":"f6667070546b139bd53a7df7d7674fd0c4250636","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T23:03:59Z","started_at":"2026-09-30T23:03:59Z","completed_at":"2026-10-01T01:28:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T01:38:40.750356717Z","timing_metadata_checked_at":"2026-10-01T01:38:40.750356717Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T01:38:40.752757409Z","region_metadata_checked_at":"2026-10-01T01:38:40.752757409Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7272/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105426318702153728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7272,"pr_state":"closed","pr_sha":"0980b4e5b970eb9dc69ab9505f060340b365390e","final_merged_sha":"0980b4e5b970eb9dc69ab9505f060340b365390e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T22:35:25Z","started_at":"2026-09-30T22:35:25Z","completed_at":"2026-10-01T01:04:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T01:08:02.801230809Z","timing_metadata_checked_at":"2026-10-01T01:08:02.801230809Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T01:08:02.804692499Z","region_metadata_checked_at":"2026-10-01T01:08:02.804692499Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"df8c4603df030a88d5df95c36cb06e1893ed209c3161baf36db8944528a92993","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7272/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105426318702153728","occurred_at":"2026-09-30T22:35:25Z","failed_at":"e2e","signature_id":"e5bf27d444d18a4f63bf57e1c97a66cb2cc56175533ecabe1444ad05ef7c4ffa","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc003c2f500\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-4t7wr4mxhmz6\" to finish creating: GET https://rp.j2153728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/911cbd71-d1cd-4d26-a932-ae7e6da8fb9f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/911cbd71-d1cd-4d26-a932-ae7e6da8fb9f\",\n      \"name\": \"911cbd71-d1cd-4d26-a932-ae7e6da8fb9f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T23:49:39.505681667Z\",\n      \"endTime\": \"2026-10-01T00:08:45.705788813Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u3l1m0a6z5n1f9b-np-deletable-265vx-bw7dl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-4t7wr4mxhmz6--managed/providers/Microsoft.Compute/virtualMachines/u3l1m0a6z5n1f9b-np-deletable-265vx-bw7dl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u3l1m0a6z5n1f9b-np-deletable-265vx-bw7dl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-4t7wr4mxhmz6--managed/providers/Microsoft.Compute/virtualMachines/u3l1m0a6z5n1f9b-np-deletable-265vx-bw7dl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc003c2f500\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-4t7wr4mxhmz6\" to finish creating: GET https://rp.j2153728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/911cbd71-d1cd-4d26-a932-ae7e6da8fb9f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/911cbd71-d1cd-4d26-a932-ae7e6da8fb9f\",\n      \"name\": \"911cbd71-d1cd-4d26-a932-ae7e6da8fb9f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T23:49:39.505681667Z\",\n      \"endTime\": \"2026-10-01T00:08:45.705788813Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u3l1m0a6z5n1f9b-np-deletable-265vx-bw7dl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-4t7wr4mxhmz6--managed/providers/Microsoft.Compute/virtualMachines/u3l1m0a6z5n1f9b-np-deletable-265vx-bw7dl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u3l1m0a6z5n1f9b-np-deletable-265vx-bw7dl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-4t7wr4mxhmz6--managed/providers/Microsoft.Compute/virtualMachines/u3l1m0a6z5n1f9b-np-deletable-265vx-bw7dl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-4t7wr4mxhmz6\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7276/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105425910554431488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7276,"pr_state":"closed","pr_sha":"efea9c8cd78bb4f9a3458bdad0eba29fd92e0ceb","final_merged_sha":"efea9c8cd78bb4f9a3458bdad0eba29fd92e0ceb","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T22:33:48Z","started_at":"2026-09-30T22:33:48Z","completed_at":"2026-10-01T00:50:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T00:57:52.444286044Z","timing_metadata_checked_at":"2026-10-01T00:57:52.444286044Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T00:57:52.447721633Z","region_metadata_checked_at":"2026-10-01T00:57:52.447721633Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"342e9dd3aba60c1c3b08bb61deab5f3deb95e74cd918ae4d6fa85d358dc8b32d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7276/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105425910554431488","occurred_at":"2026-09-30T22:33:48Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105424510625779712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6577,"pr_state":"closed","pr_sha":"3d03a40d19bf9b85b85165f4f9432cbbcf37480e","final_merged_sha":"3d03a40d19bf9b85b85165f4f9432cbbcf37480e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T22:28:14Z","started_at":"2026-09-30T22:28:14Z","completed_at":"2026-10-01T00:46:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T00:57:52.157057433Z","timing_metadata_checked_at":"2026-10-01T00:57:52.157057433Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T00:57:52.160525222Z","region_metadata_checked_at":"2026-10-01T00:57:52.160525222Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"c69b45dae2473314b7bc95fb45be89a84b49844b213b2ede53b4bba955bb87b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105424510625779712","occurred_at":"2026-09-30T22:28:14Z","failed_at":"alert","signature_id":"4fb9fed155ed0a95351ac3cfe726d666cbdd3c00e444f36f4da70034c17f36f3","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 27 time(s) (20 unknown, 7 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-9p7qd\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-9p7qd has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-75htw\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-75htw has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-hsnlr\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-hsnlr has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-bdq7h\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-bdq7h has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-t7t2f\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-t7t2f has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-c472z\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-c472z has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-6j8pl\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-6j8pl has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-5vmb4\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-5vmb4 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-z5vtc\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-z5vtc has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-9llk2\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-9llk2 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-b79df\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-b79df has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-pgcdv\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-pgcdv has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-cm978\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-cm978 has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-bb46w\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-bb46w has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-sqbrn\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-sqbrn has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-kmksg\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-kmksg has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-s5k4b\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-s5k4b has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-v55n5\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-v55n5 has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-lxq4s\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-lxq4s has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-xfjzk\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-xfjzk has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105423700407881728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6580,"pr_state":"open","pr_sha":"1fe9af59ce06c5d6f785a91421edf88db61a778f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T22:25:01Z","started_at":"2026-09-30T22:25:01Z","completed_at":"2026-10-01T01:15:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T01:18:10.855729836Z","timing_metadata_checked_at":"2026-10-01T01:18:10.855729836Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T01:18:10.859444026Z","region_metadata_checked_at":"2026-10-01T01:18:10.859444026Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"e0b6183183b94f2be4b8df3aa7e7767697c95cf4ce6304e84c77c9b7c3aa2b17","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105423700407881728","occurred_at":"2026-09-30T22:25:01Z","failed_at":"e2e","signature_id":"d39a6f967f0a8906df550b2068f509270953857462cb691e6faf8e691e932263","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:287]: failed to scale down node pool \"np-scale\" from 2 to 1 replicas\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00217d260\u003e: \n    failed waiting for nodepool=\"np-scale\" in cluster=\"priv-np-scale\" resourcegroup=\"priv-np-scale-d7bh2pbbtldm\" to finish updating, caused by: timeout '60.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-scale in cluster priv-np-scale in resource group priv-np-scale-d7bh2pbbtldm, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:287]: failed to scale down node pool \"np-scale\" from 2 to 1 replicas\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00217d260\u003e: \n    failed waiting for nodepool=\"np-scale\" in cluster=\"priv-np-scale\" resourcegroup=\"priv-np-scale-d7bh2pbbtldm\" to finish updating, caused by: timeout '60.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-scale in cluster priv-np-scale in resource group priv-np-scale-d7bh2pbbtldm, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-scale\" in cluster=\"priv-np-scale\" resourcegroup=\"priv-np-scale-d7bh2pbbtldm\" to finish updating, caused by: timeout '60.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-scale in cluster priv-np-scale in resource group priv-np-scale-d7bh2pbbtldm, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105423324162035712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7181,"pr_state":"closed","pr_sha":"1df99c98be21cd453cc4fb1e16e6b31dea03e43d","final_merged_sha":"3ae51eab9b946eb1b94407de9646f52a67175cd0","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T22:23:31Z","started_at":"2026-09-30T22:23:31Z","completed_at":"2026-10-01T00:41:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T00:47:04.169055816Z","timing_metadata_checked_at":"2026-10-01T00:47:04.169055816Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T00:47:04.173073003Z","region_metadata_checked_at":"2026-10-01T00:47:04.173073003Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"2cecefa8fdd30629cdccb1c67afa1b35a38aac5f2908474d5bef5942746fc2b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105423324162035712","occurred_at":"2026-09-30T22:23:31Z","failed_at":"alert","signature_id":"d65b8e364e6c08a1a77addfca0b9017dbd9c06c54662364ca5867cb456a5743a","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T23:47:21Z\n  Ended:       2026-09-30T23:53:25Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2035712-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5ki3fusaq8plth807pisqonnpgbdn6-e4t7z1r7k6m9l4f\", pod=\"kube-state-metrics-hcp-754f855b69-774l4\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5ki3fusaq8plth807pisqonnpgbdn6-e4t7z1r7k6m9l4f/kube-state-metrics-hcp-754f855b69-774l4 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7055/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105421859376861184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7055,"pr_state":"closed","pr_sha":"ea023d8558ae7822bf5aa0216dda1632c30fd364","final_merged_sha":"857119d4c2c287f8a23a5e84260abfca4fbbefe5","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T22:17:42Z","started_at":"2026-09-30T22:17:42Z","completed_at":"2026-10-01T00:34:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T00:36:09.102899623Z","timing_metadata_checked_at":"2026-10-01T00:36:09.102899623Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T00:36:09.105885314Z","region_metadata_checked_at":"2026-10-01T00:36:09.105885314Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105417810694901760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6993,"pr_state":"closed","pr_sha":"1a558ec553718fc2df21030c08516ca31075af74","final_merged_sha":"1a558ec553718fc2df21030c08516ca31075af74","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T22:01:37Z","started_at":"2026-09-30T22:01:37Z","completed_at":"2026-10-01T00:17:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T00:26:00.091672387Z","timing_metadata_checked_at":"2026-10-01T00:26:00.091672387Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T00:26:00.094378078Z","region_metadata_checked_at":"2026-10-01T00:26:00.094378078Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"792eec89a9c6b48446a559d6d065521860d8dcee1c0c4bbd602a126af9ad8f63","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105417810694901760","occurred_at":"2026-09-30T22:01:37Z","failed_at":"alert","signature_id":"443507f024e81fd0df310f54c24e49a4edd8dadf2d4fcc9aee122ad7ab6fba9d","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T23:20:44Z\n  Ended:       2026-09-30T23:26:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4901760-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5k6opsebogjptm5uab2qhhik4rqbt7-k4w6n5p5s8z3z6k\", pod=\"azure-file-csi-driver-controller-7c869d77d7-z84jf\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5k6opsebogjptm5uab2qhhik4rqbt7-k4w6n5p5s8z3z6k/azure-file-csi-driver-controller-7c869d77d7-z84jf has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7280/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105414632171638784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7280,"pr_state":"closed","pr_sha":"47b7fae8cc2ff19a5b122059ed0b440b2df55933","final_merged_sha":"47b7fae8cc2ff19a5b122059ed0b440b2df55933","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T21:48:59Z","started_at":"2026-09-30T21:48:59Z","completed_at":"2026-10-01T00:06:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T00:15:58.844101094Z","timing_metadata_checked_at":"2026-10-01T00:15:58.844101094Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T00:15:58.846697486Z","region_metadata_checked_at":"2026-10-01T00:15:58.846697486Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7242/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105413427336843264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7242,"pr_state":"closed","pr_sha":"136c196e662abb71c58ecca8b6794334296a7ffd","final_merged_sha":"136c196e662abb71c58ecca8b6794334296a7ffd","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T21:44:12Z","started_at":"2026-09-30T21:44:12Z","completed_at":"2026-10-01T00:09:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T00:15:58.663311453Z","timing_metadata_checked_at":"2026-10-01T00:15:58.663311453Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T00:15:58.666185045Z","region_metadata_checked_at":"2026-10-01T00:15:58.666185045Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"3969ecb7fa93087fd853295b4fb767dd44f90f125b788cb09b6f9788d873de3d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7242/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105413427336843264","occurred_at":"2026-09-30T21:44:12Z","failed_at":"e2e","signature_id":"fbd3037b7d4daa93d08a18e11bbde460acac79d80087f4b96712fcd02af72604","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b7a960\u003e: \n    failed to create HCP cluster ea-listnpdfsm: failed waiting for cluster=\"ea-listnpdfsm\" in resourcegroup=\"ea-list-g57slgdp8bj6\" to finish creating: GET https://rp.j6843264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cf1eed55-2ffc-4dee-9cff-f7085dfb10d8\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/cf1eed55-2ffc-4dee-9cff-f7085dfb10d8' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cf1eed55-2ffc-4dee-9cff-f7085dfb10d8\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b7a960\u003e: \n    failed to create HCP cluster ea-listnpdfsm: failed waiting for cluster=\"ea-listnpdfsm\" in resourcegroup=\"ea-list-g57slgdp8bj6\" to finish creating: GET https://rp.j6843264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cf1eed55-2ffc-4dee-9cff-f7085dfb10d8\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/cf1eed55-2ffc-4dee-9cff-f7085dfb10d8' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cf1eed55-2ffc-4dee-9cff-f7085dfb10d8\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound"}},{"row_id":"c3fc5fe11406b8ed678501f435a56d166537cf9c8f902d062570790192fe0cc6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7242/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105413427336843264","occurred_at":"2026-09-30T21:44:12Z","failed_at":"e2e","signature_id":"82b07f2201cf39c1115582abd5ee1c7849d4a3ff98ac6b09e90eee9b9379ac77","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cda1a0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-s9lfl9dtlsjp\" to finish creating: GET https://rp.j6843264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/165fa122-4b97-461c-a51b-3a9f4e951859\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/165fa122-4b97-461c-a51b-3a9f4e951859' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/165fa122-4b97-461c-a51b-3a9f4e951859\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cda1a0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-s9lfl9dtlsjp\" to finish creating: GET https://rp.j6843264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/165fa122-4b97-461c-a51b-3a9f4e951859\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/165fa122-4b97-461c-a51b-3a9f4e951859' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/165fa122-4b97-461c-a51b-3a9f4e951859\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7279/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105400327724339200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7279,"pr_state":"closed","pr_sha":"84cd9115e4cbbf3bca7337d7f503e7979ee88733","final_merged_sha":"84cd9115e4cbbf3bca7337d7f503e7979ee88733","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T20:52:09Z","started_at":"2026-09-30T20:52:09Z","completed_at":"2026-09-30T23:10:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T23:12:17.946396898Z","timing_metadata_checked_at":"2026-09-30T23:12:17.946396898Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T23:12:17.949673288Z","region_metadata_checked_at":"2026-09-30T23:12:17.949673288Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"962fba4f31ab6f6767f8cc9b84e9d9cae9a6cb1404ca5bdc7b09e2117437db67","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7279/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105400327724339200","occurred_at":"2026-09-30T20:52:09Z","failed_at":"e2e","signature_id":"7b187491ce9c30d3ede87c8dd648a76fcd7a020441133e8aa1cde7d221cdf9af","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-9xv2vk78f5m7\" to finish creating: GET https://rp.j4339200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f34fa79-f2bb-47f4-8491-50e2c8ff8e21\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeadlineExceeded\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f34fa79-f2bb-47f4-8491-50e2c8ff8e21\",\n  \"name\": \"9f34fa79-f2bb-47f4-8491-50e2c8ff8e21\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-30T22:15:27.259108656Z\",\n  \"endTime\": \"2026-09-30T22:34:33.885780975Z\",\n  \"error\": {\n    \"code\": \"DeadlineExceeded\",\n    \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v7o2l4s3b0x1k3m-np-update-nodes-z48kk-nvb79: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-9xv2vk78f5m7--managed/providers/Microsoft.Compute/virtualMachines/v7o2l4s3b0x1k3m-np-update-nodes-z48kk-nvb79 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v7o2l4s3b0x1k3m-np-update-nodes-z48kk-nvb79: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-9xv2vk78f5m7--managed/providers/Microsoft.Compute/virtualMachines/v7o2l4s3b0x1k3m-np-update-nodes-z48kk-nvb79 to exist\\n\"\n  }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n    \u003c[]error | len:1, cap:1\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-9xv2vk78f5m7\" to finish creating: GET https://rp.j4339200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f34fa79-f2bb-47f4-8491-50e2c8ff8e21\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeadlineExceeded\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f34fa79-f2bb-47f4-8491-50e2c8ff8e21\",\n  \"name\": \"9f34fa79-f2bb-47f4-8491-50e2c8ff8e21\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-30T22:15:27.259108656Z\",\n  \"endTime\": \"2026-09-30T22:34:33.885780975Z\",\n  \"error\": {\n    \"code\": \"DeadlineExceeded\",\n    \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v7o2l4s3b0x1k3m-np-update-nodes-z48kk-nvb79: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-9xv2vk78f5m7--managed/providers/Microsoft.Compute/virtualMachines/v7o2l4s3b0x1k3m-np-update-nodes-z48kk-nvb79 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v7o2l4s3b0x1k3m-np-update-nodes-z48kk-nvb79: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-9xv2vk78f5m7--managed/providers/Microsoft.Compute/virtualMachines/v7o2l4s3b0x1k3m-np-update-nodes-z48kk-nvb79 to exist\\n\"\n  }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n    \u003c[]error | len:1, cap:1\u003e: ...\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5","failure_pattern":"--------------------------------------------------------------------------------","search_query":"--------------------------------------------------------------------------------"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105399910143627264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6970,"pr_state":"closed","pr_sha":"af55f114edf1b1d398480eeee6a41ae706e821e2","final_merged_sha":"af55f114edf1b1d398480eeee6a41ae706e821e2","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T20:50:29Z","started_at":"2026-09-30T20:50:29Z","completed_at":"2026-09-30T23:21:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T23:23:07.98556018Z","timing_metadata_checked_at":"2026-09-30T23:23:07.98556018Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T23:23:07.98873487Z","region_metadata_checked_at":"2026-09-30T23:23:07.98873487Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105390838300282880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6976,"pr_state":"open","pr_sha":"7868ef2b26e0bfedc24919742a75cbc3e71f2259","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T20:14:26Z","started_at":"2026-09-30T20:14:26Z","completed_at":"2026-09-30T22:32:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T22:40:15.779814399Z","timing_metadata_checked_at":"2026-09-30T22:40:15.779814399Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T22:40:15.785448682Z","region_metadata_checked_at":"2026-09-30T22:40:15.785448682Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105385002328920064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6820,"pr_state":"closed","pr_sha":"c99d95a578f89e3e4c3bfc476a031bb4b59056f4","final_merged_sha":"273677ce69aa7a8dc7e1da98b8942d3ea48b24d1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T19:51:15Z","started_at":"2026-09-30T19:51:15Z","completed_at":"2026-09-30T20:59:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T21:05:43.346707467Z","timing_metadata_checked_at":"2026-09-30T21:05:43.346707467Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T21:05:43.349835457Z","region_metadata_checked_at":"2026-09-30T21:05:43.349835457Z"},"failed_at":["alert","provision"],"failed_test_count":4,"occurrences":[{"row_id":"0722757c064db6381ff80d4afa66e811aa44bd50b1cd28485390bb091f09770c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105385002328920064","occurred_at":"2026-09-30T19:51:15Z","failed_at":"alert","signature_id":"fa068afb5532b7a0878648a63fa9e7d202e29a9179905dc7ba19af246eee3c2f","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T20:39:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j8920064-svc\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"warning\"\n  Description: The Arobit forwarder metrics endpoint on cluster ci01-j8920064-svc has been unreachable for 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired"}},{"row_id":"85fb91a50462ba78d8fa9e5c07608f6a3bc7db02b136134580ec4b4e48ac5341","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105385002328920064","occurred_at":"2026-09-30T19:51:15Z","failed_at":"alert","signature_id":"4dce540a5737084b6a6f4fb2bff2bdb7b393fc2c015c007f9cfd99a7f9898533","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T20:39:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j8920064-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.33:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8920064/providers/microsoft.monitor/accounts/services-j8920064\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-6rhvw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T20:40:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j8920064-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.33:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8920064/providers/microsoft.monitor/accounts/services-j8920064\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-6rhvw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired"}},{"row_id":"995d3be8cce47fb67c1d7f254b2b07dbc1f595f3f31af76e6754f9f78acc3ac3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105385002328920064","occurred_at":"2026-09-30T19:51:15Z","failed_at":"provision","signature_id":"59c91e2e24f0ff8a4982b6cbce1ad6ea4d614e46730790c964f60065e02b662a","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","test_suite":"templatize-pipeline","failure_text":"time=2026-09-30T20:23:20.071Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-30T20:23:20.785Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j8920064-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:eb3e2e068172121a9ca133c5d17784df770dacc104f20c7d4a7df5f1c70e8a08 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:450m memory:472Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j8920064-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5c29681bb4b38d08ca21c31c4553bab72e8e9c260dc0ec875e1d014268a9e580 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:87a67f37-4acf-40f9-a2aa-33f7bdfa8ae1 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#87a67f37-4acf-40f9-a2aa-33f7bdfa8ae1 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-30T20:23:20.785Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T20:23:21.014Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-30T20:23:21.176Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-30T20:23:21.176Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T20:23:21.232Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T20:33:24.255Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T20:33:24.255Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T20:34:24.257Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-30T20:34:24.848Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j8920064-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:eb3e2e068172121a9ca133c5d17784df770dacc104f20c7d4a7df5f1c70e8a08 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:450m memory:472Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j8920064-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5c29681bb4b38d08ca21c31c4553bab72e8e9c260dc0ec875e1d014268a9e580 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:87a67f37-4acf-40f9-a2aa-33f7bdfa8ae1 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#87a67f37-4acf-40f9-a2aa-33f7bdfa8ae1 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-30T20:34:24.848Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T20:34:25.188Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-30T20:34:25.588Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T20:34:25.588Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T20:34:25.757Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-30T20:34:26.389Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T20:34:26.390Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T20:34:26.616Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T20:34:26.899Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T20:34:27.340Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T20:34:27.673Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T20:34:27.873Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T20:34:28.054Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T20:34:28.402Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-30T20:34:28.402Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T20:34:28.487Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-30T20:44:30.230Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T20:44:30.230Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T20:45:30.230Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-30T20:45:30.710Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j8920064-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:eb3e2e068172121a9ca133c5d17784df770dacc104f20c7d4a7df5f1c70e8a08 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:450m memory:472Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j8920064-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5c29681bb4b38d08ca21c31c4553bab72e8e9c260dc0ec875e1d014268a9e580 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:87a67f37-4acf-40f9-a2aa-33f7bdfa8ae1 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#87a67f37-4acf-40f9-a2aa-33f7bdfa8ae1 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-30T20:45:30.711Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T20:45:31.093Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-30T20:45:31.467Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T20:45:31.467Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T20:45:31.591Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-30T20:45:32.324Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T20:45:32.324Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T20:45:32.528Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T20:45:32.928Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T20:45:33.424Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T20:45:33.656Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T20:45:34.121Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T20:45:34.518Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T20:45:34.878Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-30T20:45:34.878Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T20:45:34.950Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-30T20:53:20.192Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"context deadline exceeded\"\ntime=2026-09-30T20:53:20.193Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","failure_pattern":"failed to roll out Helm release: context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded"}},{"row_id":"d9411080f96d720dd47fa0c8fbf8e479c8ada16c452bc602ec4613651fab823b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105385002328920064","occurred_at":"2026-09-30T19:51:15Z","failed_at":"alert","signature_id":"a917e86272dd2f9d0f31aa7b4cfdde08465b4442f6dace77ef110fd12ce6152f","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T20:29:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8920064-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-ptxgs\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-ptxgs has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T20:29:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8920064-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-rfxj6\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-rfxj6 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T20:33:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8920064-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-fwc59\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-fwc59 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105384924235173888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6993,"pr_state":"closed","pr_sha":"1a558ec553718fc2df21030c08516ca31075af74","final_merged_sha":"1a558ec553718fc2df21030c08516ca31075af74","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T19:50:56Z","started_at":"2026-09-30T19:50:56Z","completed_at":"2026-09-30T20:39:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T20:44:46.63787738Z","timing_metadata_checked_at":"2026-09-30T20:44:46.63787738Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T20:44:46.641242771Z","region_metadata_checked_at":"2026-09-30T20:44:46.641242771Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c09cbf2c3dfce92a53d3d9a1aeed7f2f194fb9514844d7aaa8d393dc3fe2fc49","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105384924235173888","occurred_at":"2026-09-30T19:50:56Z","failed_at":"provision","signature_id":"d465f96a6ebaf98458f369563694ed7b4f0235597cd9af66af75425f966c9780","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nsp (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-30T20:31:01.398Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 description=\"Step nsp\\n  Kind: ARM\\n  Template: templates/mgmt-nsp.bicep\\n  Parameters: configurations/mgmt-nsp.tmpl.bicepparam\"\ntime=2026-09-30T20:31:01.965Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2\ntime=2026-09-30T20:31:02.790Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 deployment=0a6bcb1a4c8db820be60ebb3431d76a20397d50a3cee281e89a5518a17e2b3e0 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5173888-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F0a6bcb1a4c8db820be60ebb3431d76a20397d50a3cee281e89a5518a17e2b3e0\ntime=2026-09-30T20:33:03.396Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5173888-mgmt-2/providers/Microsoft.Resources/deployments/0a6bcb1a4c8db820be60ebb3431d76a20397d50a3cee281e89a5518a17e2b3e0/operationStatuses/08584108066230621918\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5173888-mgmt-2/providers/Microsoft.Resources/deployments/profile-mlyktaqazs6u6\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5173888-mgmt-2/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j5173888-mgmt-2/resourceAssociations/nsp-j5173888-mgmt-2-ra-30be3175-d83b-506a-923e-30e4da745648\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"TaskCanceledException\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Error while processing request for association /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5173888-mgmt-2/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j5173888-mgmt-2/resourceAssociations/nsp-j5173888-mgmt-2-ra-30be3175-d83b-506a-923e-30e4da745648, Error message: The request was canceled due to the configured HttpClient.Timeout of 100 seconds elapsing.\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f315c2ea7d854194ea48cb15cb5892a7a5211cd6320329993a304b3e4444a395","failure_pattern":"ERROR CODE: DeploymentFailed; detail code TaskCanceledException; detail message Error while processing network security perimeter association: The request was canceled due to the configured HttpClient.Timeout of 100 seconds elapsing.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f315c2ea7d854194ea48cb15cb5892a7a5211cd6320329993a304b3e4444a395"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105382781734359040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7143,"pr_state":"open","pr_sha":"c44277134a494ab263b13a52300b4cbda7bd8fd2","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T19:42:25Z","started_at":"2026-09-30T19:42:25Z","completed_at":"2026-09-30T22:03:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T22:19:04.738054667Z","timing_metadata_checked_at":"2026-09-30T22:19:04.738054667Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T22:19:04.742492954Z","region_metadata_checked_at":"2026-09-30T22:19:04.742492954Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"e2050a5ba0118e5f25852a3a2793c7f261f53351b316539c9c8b7db25ce200bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105382781734359040","occurred_at":"2026-09-30T19:42:25Z","failed_at":"e2e","signature_id":"44dbc70475d1aef57e4bf508ea5b658bc75ebeac8f3ec4f8affd2ae8a9db5fcb","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:175]: cluster verification failed\nUnexpected error:\n    \u003c*errors.joinError | 0xc0015b4cc0\u003e: \n    VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: no VMs found for nodepool des-np in managed resource group des-encrypt-gngwklk9wp6n--managed\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:175]: cluster verification failed\nUnexpected error:\n    \u003c*errors.joinError | 0xc0015b4cc0\u003e: \n    VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: no VMs found for nodepool des-np in managed resource group des-encrypt-gngwklk9wp6n--managed\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a90d903a85215a711ab538a5cb9689c2473a833a5956b38bc348d987dab53d7b","failure_pattern":"VerifyVMOSDiskCustomerEncryption(nodePool=\u003cnodepool\u003e) failed: no VMs found for nodepool \u003cnodepool\u003e in managed resource group \u003cresource-group\u003e","search_query":"VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: no VMs found for nodepool des-np in managed resource group des-encrypt-gngwklk9wp6n--managed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a90d903a85215a711ab538a5cb9689c2473a833a5956b38bc348d987dab53d7b"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105370345325203456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6577,"pr_state":"closed","pr_sha":"f85307ac47d725462ca085087636a9e9fd03a66e","final_merged_sha":"3d03a40d19bf9b85b85165f4f9432cbbcf37480e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T18:52:59Z","started_at":"2026-09-30T18:52:59Z","completed_at":"2026-09-30T21:15:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T21:16:11.864014272Z","timing_metadata_checked_at":"2026-09-30T21:16:11.864014272Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T21:16:11.867277662Z","region_metadata_checked_at":"2026-09-30T21:16:11.867277662Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"1b8682400cfc92334cdeaf5257da2e85e5beb45bb2c8ee31331954a074d54b7f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105370345325203456","occurred_at":"2026-09-30T18:52:59Z","failed_at":"alert","signature_id":"1e9c1ef55ebe30e66d0a2cb5a25733429fd4759e44b43e5ffc7cc434e6416df0","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 5 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T19:59:11Z\n  Ended:       2026-09-30T20:14:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5203456-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5hgejihc22takh2irlnl38m4fd005m-q6p1l2n2a3b6z5p\", pod=\"control-plane-operator-d4ff858b6-x6hb9\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5hgejihc22takh2irlnl38m4fd005m-q6p1l2n2a3b6z5p/control-plane-operator-d4ff858b6-x6hb9 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T19:59:12Z\n  Ended:       2026-09-30T20:14:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5203456-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5hgejihc22takh2irlnl38m4fd005m-q6p1l2n2a3b6z5p\", pod=\"control-plane-operator-d4ff858b6-l2s48\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5hgejihc22takh2irlnl38m4fd005m-q6p1l2n2a3b6z5p/control-plane-operator-d4ff858b6-l2s48 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-30T19:59:12Z\n  Ended:       2026-09-30T20:14:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5203456-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5hgejihc22takh2irlnl38m4fd005m-q6p1l2n2a3b6z5p\", pod=\"control-plane-operator-d4ff858b6-4pbr7\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5hgejihc22takh2irlnl38m4fd005m-q6p1l2n2a3b6z5p/control-plane-operator-d4ff858b6-4pbr7 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-30T19:59:12Z\n  Ended:       2026-09-30T20:14:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5203456-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5hgejihc22takh2irlnl38m4fd005m-q6p1l2n2a3b6z5p\", pod=\"control-plane-operator-d4ff858b6-wwpwh\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5hgejihc22takh2irlnl38m4fd005m-q6p1l2n2a3b6z5p/control-plane-operator-d4ff858b6-wwpwh has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-30T20:00:11Z\n  Ended:       2026-09-30T20:14:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5203456-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5hgejihc22takh2irlnl38m4fd005m-q6p1l2n2a3b6z5p\", pod=\"control-plane-operator-d4ff858b6-tvjkl\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5hgejihc22takh2irlnl38m4fd005m-q6p1l2n2a3b6z5p/control-plane-operator-d4ff858b6-tvjkl has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105360937161592832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7074,"pr_state":"closed","pr_sha":"425c4c356c64b8430e722d04ae7e02d916e5788f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T18:15:37Z","started_at":"2026-09-30T18:15:37Z","completed_at":"2026-09-30T20:32:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T20:44:46.738717205Z","timing_metadata_checked_at":"2026-09-30T20:44:46.738717205Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T20:44:46.741811696Z","region_metadata_checked_at":"2026-09-30T20:44:46.741811696Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105360377792434176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7181,"pr_state":"closed","pr_sha":"c1711e8366a51583591e5ff0550d5c37cc22c519","final_merged_sha":"3ae51eab9b946eb1b94407de9646f52a67175cd0","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T18:13:24Z","started_at":"2026-09-30T18:13:24Z","completed_at":"2026-09-30T20:32:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T20:44:46.832290649Z","timing_metadata_checked_at":"2026-09-30T20:44:46.832290649Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T20:44:46.834941942Z","region_metadata_checked_at":"2026-09-30T20:44:46.834941942Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"e7967285c75d89eb2d1af2ea10f05b262bb0f8af6441b0030103b65f454d4b32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105360377792434176","occurred_at":"2026-09-30T18:13:24Z","failed_at":"e2e","signature_id":"ef889d6333cbd7f0e4691f6a9291bdf5c9b96b5e6e9bbbf10e509094968d5d90","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001666348\u003e: \n    resource group \"rg-candidate-5-1-7k2hspfgwlgc\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-1-jdl45b\" in resourcegroup=\"rg-candidate-5-1-7k2hspfgwlgc\" to finish deleting: GET https://rp.j2434176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/77026e76-9732-4942-8d30-76c6571b6048\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/77026e76-9732-4942-8d30-76c6571b6048\",\n      \"name\": \"77026e76-9732-4942-8d30-76c6571b6048\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T19:19:44.434042017Z\",\n      \"endTime\": \"2026-09-30T19:43:54.082755547Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5gr9ar4ha50b9n6jmkh1qfntpjdb09 still exists (deletion dispatched at 2026-09-30T19:19:46Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001666348\u003e: \n    resource group \"rg-candidate-5-1-7k2hspfgwlgc\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-1-jdl45b\" in resourcegroup=\"rg-candidate-5-1-7k2hspfgwlgc\" to finish deleting: GET https://rp.j2434176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/77026e76-9732-4942-8d30-76c6571b6048\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/77026e76-9732-4942-8d30-76c6571b6048\",\n      \"name\": \"77026e76-9732-4942-8d30-76c6571b6048\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T19:19:44.434042017Z\",\n      \"endTime\": \"2026-09-30T19:43:54.082755547Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5gr9ar4ha50b9n6jmkh1qfntpjdb09 still exists (deletion dispatched at 2026-09-30T19:19:46Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105355115949985792","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7053,"pr_state":"closed","pr_sha":"8470e4a8152fec1ad8e98ecdb32250dd598096a9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T17:52:29Z","started_at":"2026-09-30T17:52:29Z","completed_at":"2026-09-30T20:12:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T20:13:49.414194768Z","timing_metadata_checked_at":"2026-09-30T20:13:49.414194768Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T20:13:49.418102556Z","region_metadata_checked_at":"2026-09-30T20:13:49.418102556Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354569830633472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6820,"pr_state":"closed","pr_sha":"c99d95a578f89e3e4c3bfc476a031bb4b59056f4","final_merged_sha":"273677ce69aa7a8dc7e1da98b8942d3ea48b24d1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T17:50:19Z","started_at":"2026-09-30T17:50:19Z","completed_at":"2026-09-30T19:01:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T19:12:03.806145761Z","timing_metadata_checked_at":"2026-09-30T19:12:03.806145761Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T19:12:03.809184752Z","region_metadata_checked_at":"2026-09-30T19:12:03.809184752Z"},"failed_at":["alert","provision"],"failed_test_count":4,"occurrences":[{"row_id":"14158fca6a762126957986a1d16387e11ebf35bb70d83ddc29974a92e17ab902","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354569830633472","occurred_at":"2026-09-30T17:50:19Z","failed_at":"provision","signature_id":"7c9fdc5e068f8cbc4ff91d33bd3a3416cfd241a4e1d4d0d28ad54d1ce1f099cd","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","test_suite":"templatize-pipeline","failure_text":"time=2026-09-30T18:26:03.649Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-30T18:26:04.095Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j0633472-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:eb3e2e068172121a9ca133c5d17784df770dacc104f20c7d4a7df5f1c70e8a08 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:450m memory:472Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j0633472-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5c29681bb4b38d08ca21c31c4553bab72e8e9c260dc0ec875e1d014268a9e580 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:42f17d2a-2d54-4fa2-b94e-06bedcfbde0d region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#42f17d2a-2d54-4fa2-b94e-06bedcfbde0d INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-30T18:26:04.095Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T18:26:04.389Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-30T18:26:04.602Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-30T18:26:04.602Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T18:26:04.633Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T18:36:07.312Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T18:36:07.312Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T18:37:07.314Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-30T18:37:08.026Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j0633472-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:eb3e2e068172121a9ca133c5d17784df770dacc104f20c7d4a7df5f1c70e8a08 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:450m memory:472Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j0633472-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5c29681bb4b38d08ca21c31c4553bab72e8e9c260dc0ec875e1d014268a9e580 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:42f17d2a-2d54-4fa2-b94e-06bedcfbde0d region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#42f17d2a-2d54-4fa2-b94e-06bedcfbde0d INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-30T18:37:08.026Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T18:37:08.359Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-30T18:37:08.708Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T18:37:08.708Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T18:37:08.746Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-30T18:37:09.698Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T18:37:09.698Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T18:37:09.874Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T18:37:10.131Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T18:37:10.610Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T18:37:10.790Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T18:37:11.050Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T18:37:11.400Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T18:37:11.717Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-30T18:37:11.717Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T18:37:11.766Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-30T18:47:13.248Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T18:47:13.248Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T18:48:13.252Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-30T18:48:13.967Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j0633472-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:eb3e2e068172121a9ca133c5d17784df770dacc104f20c7d4a7df5f1c70e8a08 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:450m memory:472Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j0633472-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5c29681bb4b38d08ca21c31c4553bab72e8e9c260dc0ec875e1d014268a9e580 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:42f17d2a-2d54-4fa2-b94e-06bedcfbde0d region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#42f17d2a-2d54-4fa2-b94e-06bedcfbde0d INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-30T18:48:13.968Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T18:48:14.374Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-30T18:48:14.654Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T18:48:14.654Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T18:48:14.778Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-30T18:48:15.694Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T18:48:15.694Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T18:48:16.096Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T18:48:16.376Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T18:48:16.694Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T18:48:16.931Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T18:48:17.265Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T18:48:17.464Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T18:48:17.811Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-30T18:48:17.811Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T18:48:17.876Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-30T18:56:03.654Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"context deadline exceeded\"\ntime=2026-09-30T18:56:03.655Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","failure_pattern":"failed to roll out Helm release: context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded"}},{"row_id":"28bef2f8716812d67b6e6e25ea19c6bc93f6c2c5a7ff92017c953f4ceea28349","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354569830633472","occurred_at":"2026-09-30T17:50:19Z","failed_at":"alert","signature_id":"73766b39b2ffbd664b524f7f700b76efc77e57dbad9d236a23c2d8e65fc0b9c5","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:42:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j0633472-svc\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"warning\"\n  Description: The Arobit forwarder metrics endpoint on cluster ci01-j0633472-svc has been unreachable for 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired"}},{"row_id":"29c5a30ffae7d6e99665345e5addd700d58862d7e20b9c55632ac25b8a33f1f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354569830633472","occurred_at":"2026-09-30T17:50:19Z","failed_at":"alert","signature_id":"c7530f8f951fb0bc0dedf42f576c3e580237189727a6abff781b0abf7fea08d7","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:42:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0633472-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.92:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0633472/providers/microsoft.monitor/accounts/services-j0633472\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-kq5jf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:42:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0633472-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.92:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0633472/providers/microsoft.monitor/accounts/services-j0633472\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-kq5jf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired"}},{"row_id":"ec3893c806905c5531245d0f54e9fcc5d4e2aa61eedd85b78c44b2478b367181","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354569830633472","occurred_at":"2026-09-30T17:50:19Z","failed_at":"alert","signature_id":"032b9ffd2c84a0fa436845d9647a87344f7d1ada690aa0f9f17c34f86c623a85","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:32:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0633472-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-qp5bd\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-qp5bd has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:32:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0633472-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-x78jx\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-x78jx has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:32:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0633472-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-55p2d\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-55p2d has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354557851701248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7261,"pr_state":"closed","pr_sha":"e01d7004aa224ff873a1a813ac3e6932792e1135","final_merged_sha":"f6667070546b139bd53a7df7d7674fd0c4250636","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T17:50:16Z","started_at":"2026-09-30T17:50:16Z","completed_at":"2026-09-30T20:15:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T20:24:17.750435178Z","timing_metadata_checked_at":"2026-09-30T20:24:17.750435178Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T20:24:17.75289927Z","region_metadata_checked_at":"2026-09-30T20:24:17.75289927Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c77b3cc9745bdc781d76c5355ec6a8ce5bf9242bcd16b866a1c61d364f5088cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354557851701248","occurred_at":"2026-09-30T17:50:16Z","failed_at":"e2e","signature_id":"a106d5350ee15080f79123cc2e755f305356aa43155c5554eb5bbed8c6fbadba","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014f6680\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-mxcczjbklf6k\" to finish creating: GET https://rp.j1701248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49794eca-872f-4949-8230-a9c6d512647a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49794eca-872f-4949-8230-a9c6d512647a\",\n      \"name\": \"49794eca-872f-4949-8230-a9c6d512647a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T19:06:11.188459377Z\",\n      \"endTime\": \"2026-09-30T19:25:16.297661458Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine p5d6o0j0i5s4u7a-autoscale-noaz-686hx-dvr6p: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine p5d6o0j0i5s4u7a-autoscale-noaz-686hx-dvr6p: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014f6680\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-mxcczjbklf6k\" to finish creating: GET https://rp.j1701248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49794eca-872f-4949-8230-a9c6d512647a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49794eca-872f-4949-8230-a9c6d512647a\",\n      \"name\": \"49794eca-872f-4949-8230-a9c6d512647a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T19:06:11.188459377Z\",\n      \"endTime\": \"2026-09-30T19:25:16.297661458Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine p5d6o0j0i5s4u7a-autoscale-noaz-686hx-dvr6p: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine p5d6o0j0i5s4u7a-autoscale-noaz-686hx-dvr6p: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3a0ec083a80f290f7c051e6f9dfe7391746cc2a0f62d0ca2697a0285007d4a5","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f3a0ec083a80f290f7c051e6f9dfe7391746cc2a0f62d0ca2697a0285007d4a5"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354536905347072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6970,"pr_state":"closed","pr_sha":"af55f114edf1b1d398480eeee6a41ae706e821e2","final_merged_sha":"af55f114edf1b1d398480eeee6a41ae706e821e2","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T17:50:11Z","started_at":"2026-09-30T17:50:11Z","completed_at":"2026-09-30T20:20:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T20:24:17.473007942Z","timing_metadata_checked_at":"2026-09-30T20:24:17.473007942Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T20:24:17.476206332Z","region_metadata_checked_at":"2026-09-30T20:24:17.476206332Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"1c6e9e913497fc9db7a8b9f4487cd7d0d87ab4e1c2dc1aaa6c7ed856afe9b538","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354536905347072","occurred_at":"2026-09-30T17:50:11Z","failed_at":"e2e","signature_id":"32302e8488c217b3d6d677ca11a50f92f8cc2fff814de0e9b445cdfbf18e3592","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc000e5b548\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-46kd7.apps.aro.cilium-cl.f3k6.j5347072.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-46kd7.apps.aro.cilium-cl.f3k6.j5347072.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-46kd7.apps.aro.cilium-cl.f3k6.j5347072.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc000e5b548\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-46kd7.apps.aro.cilium-cl.f3k6.j5347072.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-46kd7.apps.aro.cilium-cl.f3k6.j5347072.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-46kd7.apps.aro.cilium-cl.f3k6.j5347072.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-46kd7.apps.aro.cilium-cl.f3k6.j5347072.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-46kd7.apps.aro.cilium-cl.f3k6.j5347072.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-46kd7.apps.aro.cilium-cl.f3k6.j5347072.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"566ab055016e04a6701fbec38885336198599e54140b613d9b347e299b8a5fee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354536905347072","occurred_at":"2026-09-30T17:50:11Z","failed_at":"alert","signature_id":"1b88ffe98af3a32b79edf3e358fff5366d78c3eb3856b592d15721147e9bc14e","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T19:14:32Z\n  Ended:       2026-09-30T19:40:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j5347072-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.33:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5347072/providers/microsoft.monitor/accounts/hcps-j5347072\", namespace=\"ocm-arohcpci01-2t5gnhij1pbtee5r1icg9qlas1lv2h5j-cilium-cluster\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"009dd713-cc58-4cfb-b82f-f1fca7b413d6\"\n  Description: Pod ocm-arohcpci01-2t5gnhij1pbtee5r1icg9qlas1lv2h5j-cilium-cluster/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T19:14:32Z\n  Ended:       2026-09-30T19:41:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j5347072-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.33:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5347072/providers/microsoft.monitor/accounts/hcps-j5347072\", namespace=\"ocm-arohcpci01-2t5gnhij1pbtee5r1icg9qlas1lv2h5j-cilium-cluster\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"009dd713-cc58-4cfb-b82f-f1fca7b413d6\"\n  Description: Pod ocm-arohcpci01-2t5gnhij1pbtee5r1icg9qlas1lv2h5j-cilium-cluster/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"6135af3e05ea7f621c994c5205e2c81597fd1be72ef583fa17eb41bdf2f67830","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354536905347072","occurred_at":"2026-09-30T17:50:11Z","failed_at":"e2e","signature_id":"f4f59b5ba44666f24d23e97f157df80ef1f26ced02f1c0c9bfaab2ad9f5e855b","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001128760\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-h59xkph9dfpm\" to finish creating: GET https://rp.j5347072.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/88da31a3-069e-4240-80a2-676685212f2c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/88da31a3-069e-4240-80a2-676685212f2c\",\n      \"name\": \"88da31a3-069e-4240-80a2-676685212f2c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T18:49:41.355729108Z\",\n      \"endTime\": \"2026-09-30T19:18:42.930044888Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001128760\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-h59xkph9dfpm\" to finish creating: GET https://rp.j5347072.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/88da31a3-069e-4240-80a2-676685212f2c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/88da31a3-069e-4240-80a2-676685212f2c\",\n      \"name\": \"88da31a3-069e-4240-80a2-676685212f2c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T18:49:41.355729108Z\",\n      \"endTime\": \"2026-09-30T19:18:42.930044888Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8404bee177ab7afaec301e103109170a41a02e3a81b48d821068765798f1085d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354536905347072","occurred_at":"2026-09-30T17:50:11Z","failed_at":"alert","signature_id":"dabbf455e614a0564c0abb3cb0bd1b76792259597e5b2a81a72dd954978bbab0","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T19:11:34Z\n  Ended:       2026-09-30T19:36:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j5347072-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.33:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5347072/providers/microsoft.monitor/accounts/hcps-j5347072\", namespace=\"ocm-arohcpci01-2t5gnhij1pbtee5r1icg9qlas1lv2h5j-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-72jx2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t5gnhij1pbtee5r1icg9qlas1lv2h5j-cilium-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T19:22:36Z\n  Ended:       2026-09-30T19:36:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j5347072-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.33:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5347072/providers/microsoft.monitor/accounts/hcps-j5347072\", namespace=\"ocm-arohcpci01-2t5gnhij1pbtee5r1icg9qlas1lv2h5j-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-72jx2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t5gnhij1pbtee5r1icg9qlas1lv2h5j-cilium-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"e42cf3abc9f03f5b7b9e323f1771245d9d08937d0afa4e25f7310c551f77f01e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354536905347072","occurred_at":"2026-09-30T17:50:11Z","failed_at":"alert","signature_id":"36bab4fa8a374fa0764d196f80453ffc18fe93b52d905fcc5c5da63a9d7c7d96","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T19:25:32Z\n  Ended:       2026-09-30T19:38:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5347072-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.33:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5347072/providers/microsoft.monitor/accounts/hcps-j5347072\", namespace=\"ocm-arohcpci01-2t5gnhij1pbtee5r1icg9qlas1lv2h5j-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-72jx2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5gnhij1pbtee5r1icg9qlas1lv2h5j-cilium-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T19:25:32Z\n  Ended:       2026-09-30T19:38:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5347072-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.33:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5347072/providers/microsoft.monitor/accounts/hcps-j5347072\", namespace=\"ocm-arohcpci01-2t5gnhij1pbtee5r1icg9qlas1lv2h5j-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-72jx2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5gnhij1pbtee5r1icg9qlas1lv2h5j-cilium-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354517317947392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6900,"pr_state":"closed","pr_sha":"7551c0170c5cffea9e64f3707dcf2378a40b7d3f","final_merged_sha":"d63de0c62ddb9d7d7de369b28c8d93807d13672c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T17:50:07Z","started_at":"2026-09-30T17:50:07Z","completed_at":"2026-09-30T18:10:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T18:29:07.480883426Z","timing_metadata_checked_at":"2026-09-30T18:29:07.480883426Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-30T18:29:07.486962108Z","region_metadata_checked_at":"2026-09-30T19:32:30.846466611Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"fa7a70ab2f51e58bce1f4a1e8bee6dab536edb6e683c9d8af5536f7a1bc47862","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354517317947392","occurred_at":"2026-09-30T17:50:07Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354426800672768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7141,"pr_state":"closed","pr_sha":"6afba8239efe6245441f25e86ccd2ab7ba7f5035","final_merged_sha":"6afba8239efe6245441f25e86ccd2ab7ba7f5035","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T17:49:45Z","started_at":"2026-09-30T17:49:45Z","completed_at":"2026-09-30T20:23:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T20:24:17.636537032Z","timing_metadata_checked_at":"2026-09-30T20:24:17.636537032Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T20:24:17.639296924Z","region_metadata_checked_at":"2026-09-30T20:24:17.639296924Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105349698964426752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6580,"pr_state":"open","pr_sha":"4aa6a298e0b5fbbd9a3319bd9a8e7be421dc6b6b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T17:30:58Z","started_at":"2026-09-30T17:30:58Z","completed_at":"2026-09-30T19:49:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T19:53:12.389127734Z","timing_metadata_checked_at":"2026-09-30T19:53:12.389127734Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T19:53:12.391933027Z","region_metadata_checked_at":"2026-09-30T19:53:12.391933027Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"1bfd92995ffc627a5bcbe66840c6bf30efbf0a71b9da37c7bce568acafa17f69","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105349698964426752","occurred_at":"2026-09-30T17:30:58Z","failed_at":"alert","signature_id":"2850f1981822705d765169c8df6b099246627ba71fa905aafb70cde8db42aecd","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T18:52:59Z\n  Ended:       2026-09-30T19:08:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j4426752-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.111:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4426752/providers/microsoft.monitor/accounts/hcps-j4426752\", namespace=\"ocm-arohcpci01-2t5ga8d53uj3o7nh34nh77qbpeiclu0r-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-72qr2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t5ga8d53uj3o7nh34nh77qbpeiclu0r-sre-hcp-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T18:52:59Z\n  Ended:       2026-09-30T19:08:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j4426752-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.111:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4426752/providers/microsoft.monitor/accounts/hcps-j4426752\", namespace=\"ocm-arohcpci01-2t5ga8d53uj3o7nh34nh77qbpeiclu0r-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-72qr2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t5ga8d53uj3o7nh34nh77qbpeiclu0r-sre-hcp-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"267cee4456e6c477ccdbe87f52940b12a2799e6a4afe3ae2140c814e0662f13d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105349698964426752","occurred_at":"2026-09-30T17:30:58Z","failed_at":"alert","signature_id":"27312778a2840cef9f5cf9f4455e8cbaa9b89abd17091183ef5636ac2a6c1aae","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T18:43:56Z\n  Ended:       2026-09-30T19:12:57Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j4426752-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.111:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4426752/providers/microsoft.monitor/accounts/hcps-j4426752\", namespace=\"ocm-arohcpci01-2t5ga8d53uj3o7nh34nh77qbpeiclu0r-sre-hcp-cluster\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"bce6bb08-1647-4613-853e-b0a5759423ff\"\n  Description: Pod ocm-arohcpci01-2t5ga8d53uj3o7nh34nh77qbpeiclu0r-sre-hcp-cluster/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T18:44:57Z\n  Ended:       2026-09-30T19:12:57Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j4426752-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.111:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4426752/providers/microsoft.monitor/accounts/hcps-j4426752\", namespace=\"ocm-arohcpci01-2t5ga8d53uj3o7nh34nh77qbpeiclu0r-sre-hcp-cluster\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"bce6bb08-1647-4613-853e-b0a5759423ff\"\n  Description: Pod ocm-arohcpci01-2t5ga8d53uj3o7nh34nh77qbpeiclu0r-sre-hcp-cluster/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"356bcc259121de6bd3c43a3674190686c2ba703112074cf43c432506d492b083","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105349698964426752","occurred_at":"2026-09-30T17:30:58Z","failed_at":"e2e","signature_id":"67c7217815599c64efc725333fb06153f8b3a9a0dd9d914a2a506c84453c1f10","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00120c640\u003e: \n    failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-mnp9svrwqcpp\" to finish creating: GET https://rp.j4426752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fd78a0f-65d2-4ffb-8dde-60a82af78dd0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fd78a0f-65d2-4ffb-8dde-60a82af78dd0\",\n      \"name\": \"9fd78a0f-65d2-4ffb-8dde-60a82af78dd0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T18:21:17.653151468Z\",\n      \"endTime\": \"2026-09-30T18:50:22.965863416Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00120c640\u003e: \n    failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-mnp9svrwqcpp\" to finish creating: GET https://rp.j4426752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fd78a0f-65d2-4ffb-8dde-60a82af78dd0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fd78a0f-65d2-4ffb-8dde-60a82af78dd0\",\n      \"name\": \"9fd78a0f-65d2-4ffb-8dde-60a82af78dd0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T18:21:17.653151468Z\",\n      \"endTime\": \"2026-09-30T18:50:22.965863416Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"5bf47199bedc79b5cd3953894021668017d49701cfec73096b581d00fb1e2f6a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105349698964426752","occurred_at":"2026-09-30T17:30:58Z","failed_at":"alert","signature_id":"d640d21aa16253cbebe72dcc2119bb10a7a01ae60ba5700101a61e6f4ab54852","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T18:57:01Z\n  Ended:       2026-09-30T19:10:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4426752-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.111:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4426752/providers/microsoft.monitor/accounts/hcps-j4426752\", namespace=\"ocm-arohcpci01-2t5ga8d53uj3o7nh34nh77qbpeiclu0r-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-72qr2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5ga8d53uj3o7nh34nh77qbpeiclu0r-sre-hcp-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T18:57:01Z\n  Ended:       2026-09-30T19:10:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4426752-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.111:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4426752/providers/microsoft.monitor/accounts/hcps-j4426752\", namespace=\"ocm-arohcpci01-2t5ga8d53uj3o7nh34nh77qbpeiclu0r-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-72qr2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5ga8d53uj3o7nh34nh77qbpeiclu0r-sre-hcp-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"f0b04248e5a2d8efadd4fe166c99c547bc35e59d6117b6fa535e391c14056985","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105349698964426752","occurred_at":"2026-09-30T17:30:58Z","failed_at":"e2e","signature_id":"5516c4e0d66f0bcaac8052c3730dabeb8f387df4e3055ca5a2bb6566c65eeb38","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:226]: Timed out after 300.001s.\nKAS→kubelet log path never became reachable from VM inside the VNet\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:220 with:\nno running pods found in openshift-kube-apiserver\nExpected\n    \u003cstring\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:226]: Timed out after 300.001s.\nKAS→kubelet log path never became reachable from VM inside the VNet\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:220 with:\nno running pods found in openshift-kube-apiserver\nExpected\n    \u003cstring\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0d4e0f7e2fc9458905eb66d446ace1531b88fae886b89c70332091e7cd2070cd","failure_pattern":"no running pods found in openshift-kube-apiserver","search_query":"no running pods found in openshift-kube-apiserver"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["0d4e0f7e2fc9458905eb66d446ace1531b88fae886b89c70332091e7cd2070cd","13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7258/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105348343122104320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7258,"pr_state":"closed","pr_sha":"8f1369d67221dfa00a0bb0152a41464a28923688","final_merged_sha":"a68022814b830b9a8334e00ff48929e4987b9c4a","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T17:25:35Z","started_at":"2026-09-30T17:25:35Z","completed_at":"2026-09-30T19:53:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T20:13:49.840620519Z","timing_metadata_checked_at":"2026-09-30T20:13:49.840620519Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T20:13:49.843437211Z","region_metadata_checked_at":"2026-09-30T20:13:49.843437211Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105342498506280960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6993,"pr_state":"closed","pr_sha":"1a558ec553718fc2df21030c08516ca31075af74","final_merged_sha":"1a558ec553718fc2df21030c08516ca31075af74","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T17:02:21Z","started_at":"2026-09-30T17:02:21Z","completed_at":"2026-09-30T19:14:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T19:22:23.075444721Z","timing_metadata_checked_at":"2026-09-30T19:22:23.075444721Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T19:22:23.078359112Z","region_metadata_checked_at":"2026-09-30T19:22:23.078359112Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"7b1561121e2386a6ad753da0ba9bc1fd5adf3f6e94fb4bba0b75cbd7d93a0da0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105342498506280960","occurred_at":"2026-09-30T17:02:21Z","failed_at":"alert","signature_id":"43d2c6ba277aa8bbbc5cc8bc7b5f2aa98baae126a8f1b3317711f9e7e5ae87d9","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T17:57:46Z\n  Ended:       2026-09-30T18:21:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6280960-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5fn1t8q899f67e9c3gfapqs0nka9q4-z9j3x8s3e4f8g8i\", pod=\"azure-file-csi-driver-operator-6fc68567db-xb6kc\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5fn1t8q899f67e9c3gfapqs0nka9q4-z9j3x8s3e4f8g8i/azure-file-csi-driver-operator-6fc68567db-xb6kc has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T17:57:46Z\n  Ended:       2026-09-30T18:21:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6280960-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5fn1t8q899f67e9c3gfapqs0nka9q4-z9j3x8s3e4f8g8i\", pod=\"azure-file-csi-driver-operator-6fc68567db-ck5hv\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5fn1t8q899f67e9c3gfapqs0nka9q4-z9j3x8s3e4f8g8i/azure-file-csi-driver-operator-6fc68567db-ck5hv has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7271/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105340384333795328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7271,"pr_state":"open","pr_sha":"0e6699ba542781a024685f4459df8366ac7e7758","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T16:53:57Z","started_at":"2026-09-30T16:53:57Z","completed_at":"2026-09-30T19:09:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T19:12:04.050578011Z","timing_metadata_checked_at":"2026-09-30T19:12:04.050578011Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T19:12:04.053520602Z","region_metadata_checked_at":"2026-09-30T19:12:04.053520602Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"76c8cf1608edddbc29c933ced50fbec3c406eea6fad1853a341c19c22b4c9c58","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7271/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105340384333795328","occurred_at":"2026-09-30T16:53:57Z","failed_at":"alert","signature_id":"c43369e4c310c6727d22ed8986f0196b36cfda0efe919bb5036a8c8d32b2a996","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T17:53:32Z\n  Ended:       2026-09-30T18:09:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3795328-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T17:56:30Z\n  Ended:       2026-09-30T18:09:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3795328-svc\", component=\"backend\", name=\"clusterclusterserviceupdatedispatch\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue clusterclusterserviceupdatedispatch has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-30T17:59:28Z\n  Ended:       2026-09-30T18:09:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3795328-svc\", component=\"backend\", name=\"csstatedump\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105335648247353344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6807,"pr_state":"open","pr_sha":"e0f7c1daf2fdc08782c031a5b04a7bc7fcfc0c90","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T16:35:08Z","started_at":"2026-09-30T16:35:08Z","completed_at":"2026-09-30T18:56:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T19:01:24.217237684Z","timing_metadata_checked_at":"2026-09-30T19:01:24.217237684Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T19:01:24.220514676Z","region_metadata_checked_at":"2026-09-30T19:01:24.220514676Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105334781733507072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6884,"pr_state":"closed","pr_sha":"50a683fcdea603a7e5aec2dbc6a88e41a26babc3","final_merged_sha":"d98ef6349a3cd6d57591590dbb9cbff37ac89433","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T16:31:41Z","started_at":"2026-09-30T16:31:41Z","completed_at":"2026-09-30T18:47:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T18:50:38.747004045Z","timing_metadata_checked_at":"2026-09-30T18:50:38.747004045Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T18:50:38.750062536Z","region_metadata_checked_at":"2026-09-30T18:50:38.750062536Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105331328655298560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7124,"pr_state":"closed","pr_sha":"60e1aa40eaea69e03059bb27ebcf9932e637a29a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T16:17:58Z","started_at":"2026-09-30T16:17:58Z","completed_at":"2026-09-30T18:27:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T18:39:57.236875773Z","timing_metadata_checked_at":"2026-09-30T18:39:57.236875773Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T18:39:57.24109076Z","region_metadata_checked_at":"2026-09-30T18:39:57.24109076Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"18be967efb11aaa4a6d65c45053c32e6308b3dd467a4a5ce9fe5869c24c11e86","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105331328655298560","occurred_at":"2026-09-30T16:17:58Z","failed_at":"alert","signature_id":"a0c6a2c52a8e1e3c5b6144fde866a89d2692c5d407e38e01f2154424838e188d","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 13 time(s) (1 unknown, 12 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T17:13:21Z\n  Ended:       2026-09-30T17:39:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5298560-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f514d5mgc9haie801lsop9ne436rb-q0n1v0q8g3o5z5q\", pod=\"router-8445c79445-9rnr2\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f514d5mgc9haie801lsop9ne436rb-q0n1v0q8g3o5z5q/router-8445c79445-9rnr2 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"457f30ea1c767424543b62f81eb28c323addd88bbde44ce0671f5ee57f5c75cb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105331328655298560","occurred_at":"2026-09-30T16:17:58Z","failed_at":"e2e","signature_id":"65e42ab979836c2386f9206947aca6b2f3f248b30240970282f08d100e07f667","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-1-z8sw852rvcl6/cluster-candidate-5-1-f8bld9 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f24640\u003e: \n    failed to create HCP cluster cluster-candidate-5-1-f8bld9: failed waiting for cluster=\"cluster-candidate-5-1-f8bld9\" in resourcegroup=\"rg-candidate-5-1-z8sw852rvcl6\" to finish creating: GET https://rp.j5298560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1da1b003-1c66-47c8-99b2-f49235ea2736\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1da1b003-1c66-47c8-99b2-f49235ea2736\",\n      \"name\": \"1da1b003-1c66-47c8-99b2-f49235ea2736\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:01:56.141378012Z\",\n      \"endTime\": \"2026-09-30T17:30:57.762149233Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-ec.1 is Partial (want Completed), started 24m10s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-1-z8sw852rvcl6/cluster-candidate-5-1-f8bld9 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f24640\u003e: \n    failed to create HCP cluster cluster-candidate-5-1-f8bld9: failed waiting for cluster=\"cluster-candidate-5-1-f8bld9\" in resourcegroup=\"rg-candidate-5-1-z8sw852rvcl6\" to finish creating: GET https://rp.j5298560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1da1b003-1c66-47c8-99b2-f49235ea2736\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1da1b003-1c66-47c8-99b2-f49235ea2736\",\n      \"name\": \"1da1b003-1c66-47c8-99b2-f49235ea2736\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:01:56.141378012Z\",\n      \"endTime\": \"2026-09-30T17:30:57.762149233Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-ec.1 is Partial (want Completed), started 24m10s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105330420139692032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7068,"pr_state":"closed","pr_sha":"269e5d56bcfac6dc9f8d3ca87b7fb1ea84bf53b2","final_merged_sha":"269e5d56bcfac6dc9f8d3ca87b7fb1ea84bf53b2","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T16:14:20Z","started_at":"2026-09-30T16:14:20Z","completed_at":"2026-09-30T18:42:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T18:50:38.978983059Z","timing_metadata_checked_at":"2026-09-30T18:50:38.978983059Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T18:50:38.981589351Z","region_metadata_checked_at":"2026-09-30T18:50:38.981589351Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7178,"pr_state":"open","pr_sha":"6f8eb9e8dcaa57d6bdd65df600b63d5e8d60c3a9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T16:03:39Z","started_at":"2026-09-30T16:03:39Z","completed_at":"2026-09-30T18:34:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T18:50:39.153609543Z","timing_metadata_checked_at":"2026-09-30T18:50:39.153609543Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T18:50:39.156400535Z","region_metadata_checked_at":"2026-09-30T18:50:39.156400535Z"},"failed_at":["alert","e2e"],"failed_test_count":23,"occurrences":[{"row_id":"01cd1bd1a0e51da6c975e3de44ef08f2ccaaa72333ccf4360021bb2b20b01d43","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"alert","signature_id":"5fddfe712c98fb0e643ec5bb4c03b94974770b44c356b20f027ad56567a251f3","test_name":"[aro-hcp-observability] [svc] alert KubeNodeUnreachable does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T17:15:29Z\n  Ended:       2026-09-30T17:26:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/services-j5446016\", namespace=\"prometheus\", node=\"aks-userswft1-23433410-vmss000001\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: aks-userswft1-23433410-vmss000001 is unreachable and some workloads may be rescheduled.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T17:16:26Z\n  Ended:       2026-09-30T17:26:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/services-j5446016\", namespace=\"prometheus\", node=\"aks-userswft1-23433410-vmss000001\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: aks-userswft1-23433410-vmss000001 is unreachable and some workloads may be rescheduled.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"519fe0fe7478bebfb7ef939c84f76fa0281886d85cb376148838a1878147b618","failure_pattern":"alert [svc] KubeNodeUnreachable fired","search_query":"alert [svc] KubeNodeUnreachable fired"}},{"row_id":"04f5f2b3781044e3f493d68ba681e321d09abc88943822ede4f3964816b07718","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"e1a6da9fb27aa2281cf1d566ff4e3821f5a89e8834f371968aac14a196fb4167","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00104c300\u003e: \n    resource group \"nodepool-delete-c4hb66lp5frs\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-c4hb66lp5frs\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e6cc5ae-d28a-4989-ad49-e0dd769288e5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e6cc5ae-d28a-4989-ad49-e0dd769288e5\",\n      \"name\": \"7e6cc5ae-d28a-4989-ad49-e0dd769288e5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:25:05.3374218Z\",\n      \"endTime\": \"2026-09-30T17:49:05.822251371Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f18hpo95treletippqdt0q6nd6217 still exists (deletion dispatched at 2026-09-30T17:25:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00104c300\u003e: \n    resource group \"nodepool-delete-c4hb66lp5frs\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-c4hb66lp5frs\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e6cc5ae-d28a-4989-ad49-e0dd769288e5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e6cc5ae-d28a-4989-ad49-e0dd769288e5\",\n      \"name\": \"7e6cc5ae-d28a-4989-ad49-e0dd769288e5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:25:05.3374218Z\",\n      \"endTime\": \"2026-09-30T17:49:05.822251371Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f18hpo95treletippqdt0q6nd6217 still exists (deletion dispatched at 2026-09-30T17:25:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0b8bc93626235d4cb9fc042f927def82b4b8c61f1df6f1d07b269cb961f65adc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"alert","signature_id":"0e1182513eb29f65f3a7935b97b78d6d7dd25e1ef38c392477b0b0539d97cd71","test_name":"[aro-hcp-observability] [hcp] alert HCPClusterOperatorUnavailable does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T17:40:28Z\n  Ended:       2026-09-30T18:18:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0hjlm2j48ooq92uj0su5rbnsq03t-x3y7i6e5z1z9c0c\", region=\"westus3\", severity=\"warning\"\n  Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2t5f0hjlm2j48ooq92uj0su5rbnsq03t-x3y7i6e5z1z9c0c (management cluster ci01-j5446016-mgmt-1) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c65eaecf2bcf9cab816e9e9171a4f334676466d8dfb2f88bb604984fc8e6cfe2","failure_pattern":"alert [hcp] HCPClusterOperatorUnavailable fired","search_query":"alert [hcp] HCPClusterOperatorUnavailable fired"}},{"row_id":"161e5eefe0ad127bda4f2e3d03f6fb4b357e2d45fce6bbb4f54017a825c72366","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"7eadcca87592abc851861d4b05dac9a3fd2f7a948247bc541b68859484eb6b48","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00171ee58\u003e: \n    resource group \"v20261001preview-8xzpnp7tdgz2\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-8xzpnp7tdgz2\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b4e803e-d3df-4fa2-8617-7c767d9fdb48\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b4e803e-d3df-4fa2-8617-7c767d9fdb48\",\n      \"name\": \"7b4e803e-d3df-4fa2-8617-7c767d9fdb48\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:20:45.793371573Z\",\n      \"endTime\": \"2026-09-30T17:44:45.916509607Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f0lc5egjrilphmb608mchbeokkhli still exists (deletion dispatched at 2026-09-30T17:20:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00171ee58\u003e: \n    resource group \"v20261001preview-8xzpnp7tdgz2\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-8xzpnp7tdgz2\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b4e803e-d3df-4fa2-8617-7c767d9fdb48\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b4e803e-d3df-4fa2-8617-7c767d9fdb48\",\n      \"name\": \"7b4e803e-d3df-4fa2-8617-7c767d9fdb48\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:20:45.793371573Z\",\n      \"endTime\": \"2026-09-30T17:44:45.916509607Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f0lc5egjrilphmb608mchbeokkhli still exists (deletion dispatched at 2026-09-30T17:20:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"176744d5c830be212abb122f4d0ed0fd6dbf2195aae13f216403ed952b53ca6d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"0a60267a540ef98cc9481c666f895630a98788d08afe3a5800859b181c2717b2","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000af62c0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-fs49jjvbxrlk\" to finish creating: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e9c1f57e-28e9-49ea-8226-45b731c2d8ca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e9c1f57e-28e9-49ea-8226-45b731c2d8ca\",\n      \"name\": \"e9c1f57e-28e9-49ea-8226-45b731c2d8ca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:53:07.232391508Z\",\n      \"endTime\": \"2026-09-30T17:22:15.941466136Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 25m42s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000af62c0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-fs49jjvbxrlk\" to finish creating: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e9c1f57e-28e9-49ea-8226-45b731c2d8ca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e9c1f57e-28e9-49ea-8226-45b731c2d8ca\",\n      \"name\": \"e9c1f57e-28e9-49ea-8226-45b731c2d8ca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:53:07.232391508Z\",\n      \"endTime\": \"2026-09-30T17:22:15.941466136Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 25m42s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1df8570fe4ddd30d7950398a45af3c00aeeef658cb5f618edc4bb275b9484eb9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"alert","signature_id":"94689deb46415801292ac0e274c7ee723941c2649c96c6c36d110ba269a8a320","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetMisScheduled does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T17:16:20Z\n  Ended:       2026-09-30T17:27:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/services-j5446016\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet velero/node-agent are running where they are not supposed to run.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T17:16:20Z\n  Ended:       2026-09-30T17:27:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/services-j5446016\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet arobit/arobit-forwarder are running where they are not supposed to run.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-30T17:16:20Z\n  Ended:       2026-09-30T17:26:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/services-j5446016\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet arobit/arobit-forwarder are running where they are not supposed to run.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-30T17:16:20Z\n  Ended:       2026-09-30T17:26:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/services-j5446016\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet velero/node-agent are running where they are not supposed to run.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e4f1a1ea46940043a3767809b952d9be40a221815d3f81ceb038491c52599831","failure_pattern":"alert [svc] KubeDaemonSetMisScheduled fired","search_query":"alert [svc] KubeDaemonSetMisScheduled fired"}},{"row_id":"22c603dee53dcfad2f214c15bb01b8a02db7128ca89c0e8a43ca2e19b9390f7f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"9819845289ed857fc47e38f741826a409f9a62a82fd091add158c84457fae98f","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc002518090\u003e: \n    resource group \"node-limit-test-2jbt488tkpl7\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-2jbt488tkpl7\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da56d4ee-d492-4bc3-b379-32ba10c1db68\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da56d4ee-d492-4bc3-b379-32ba10c1db68\",\n      \"name\": \"da56d4ee-d492-4bc3-b379-32ba10c1db68\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:02:22.062008253Z\",\n      \"endTime\": \"2026-09-30T17:26:25.935485649Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f1mqu517if66tcq7vv1883rm6svr2 still exists (deletion dispatched at 2026-09-30T17:02:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc002518090\u003e: \n    resource group \"node-limit-test-2jbt488tkpl7\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-2jbt488tkpl7\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da56d4ee-d492-4bc3-b379-32ba10c1db68\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da56d4ee-d492-4bc3-b379-32ba10c1db68\",\n      \"name\": \"da56d4ee-d492-4bc3-b379-32ba10c1db68\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:02:22.062008253Z\",\n      \"endTime\": \"2026-09-30T17:26:25.935485649Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f1mqu517if66tcq7vv1883rm6svr2 still exists (deletion dispatched at 2026-09-30T17:02:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2636bbb6c17309d38e7b515826e59b6a3710754313e023bd18317ae3d5124860","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"80e82fe0ee940a917440e65af57e56d56e909ddf306b97e303c0192cedaf42f9","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001b32210\u003e: \n    resource group \"rg-candidate-4-21-fjd2gblvhgpk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-jb8n5l\" in resourcegroup=\"rg-candidate-4-21-fjd2gblvhgpk\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cbfc4ac-c696-45d2-b553-5a272022650d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cbfc4ac-c696-45d2-b553-5a272022650d\",\n      \"name\": \"9cbfc4ac-c696-45d2-b553-5a272022650d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:14:14.963393723Z\",\n      \"endTime\": \"2026-09-30T17:38:16.020897196Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f0e8dbaqbo8t5dt5vdcasi8k08a4q still exists (deletion dispatched at 2026-09-30T17:14:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001b32210\u003e: \n    resource group \"rg-candidate-4-21-fjd2gblvhgpk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-jb8n5l\" in resourcegroup=\"rg-candidate-4-21-fjd2gblvhgpk\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cbfc4ac-c696-45d2-b553-5a272022650d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cbfc4ac-c696-45d2-b553-5a272022650d\",\n      \"name\": \"9cbfc4ac-c696-45d2-b553-5a272022650d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:14:14.963393723Z\",\n      \"endTime\": \"2026-09-30T17:38:16.020897196Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f0e8dbaqbo8t5dt5vdcasi8k08a4q still exists (deletion dispatched at 2026-09-30T17:14:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2a3da94daddf8b1ccb6eb62aaba296e48b5425986f4a28d44a24931db1d2e345","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"9980c378ec40833f79eaebfd6e66376e813784c1f455cf437569399d4dfcefb8","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00280c2a0\u003e: \n    resource group \"rg-candidate-5-0-ns2cs4dsdbxw\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-spnhk7\" in resourcegroup=\"rg-candidate-5-0-ns2cs4dsdbxw\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c7972329-b9a1-4d78-b6ee-4512a34c9cf8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c7972329-b9a1-4d78-b6ee-4512a34c9cf8\",\n      \"name\": \"c7972329-b9a1-4d78-b6ee-4512a34c9cf8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:20:04.153634774Z\",\n      \"endTime\": \"2026-09-30T17:44:05.883180823Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f0ebuj103h4dn4jek0oofbev66a3k still exists (deletion dispatched at 2026-09-30T17:20:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00280c2a0\u003e: \n    resource group \"rg-candidate-5-0-ns2cs4dsdbxw\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-spnhk7\" in resourcegroup=\"rg-candidate-5-0-ns2cs4dsdbxw\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c7972329-b9a1-4d78-b6ee-4512a34c9cf8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c7972329-b9a1-4d78-b6ee-4512a34c9cf8\",\n      \"name\": \"c7972329-b9a1-4d78-b6ee-4512a34c9cf8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:20:04.153634774Z\",\n      \"endTime\": \"2026-09-30T17:44:05.883180823Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f0ebuj103h4dn4jek0oofbev66a3k still exists (deletion dispatched at 2026-09-30T17:20:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2e40c647adb2db4e6382852fc257ee6c9cadbae63e46de502a2b332806252618","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"b8d7fd2de97ced8753171c37ab1fa36055ef2afb986b3a9217669c39fc61ff3d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-1-9z27ks86nc27/cluster-candidate-5-1-hrwtgg should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010a04c0\u003e: \n    failed to create HCP cluster cluster-candidate-5-1-hrwtgg: failed waiting for cluster=\"cluster-candidate-5-1-hrwtgg\" in resourcegroup=\"rg-candidate-5-1-9z27ks86nc27\" to finish creating: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9135939f-e992-427f-be1e-4ecfae2b319a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9135939f-e992-427f-be1e-4ecfae2b319a\",\n      \"name\": \"9135939f-e992-427f-be1e-4ecfae2b319a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:52:29.681806994Z\",\n      \"endTime\": \"2026-09-30T17:21:35.77427567Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-ec.1 is Partial (want Completed), started 22m59s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-1-9z27ks86nc27/cluster-candidate-5-1-hrwtgg should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010a04c0\u003e: \n    failed to create HCP cluster cluster-candidate-5-1-hrwtgg: failed waiting for cluster=\"cluster-candidate-5-1-hrwtgg\" in resourcegroup=\"rg-candidate-5-1-9z27ks86nc27\" to finish creating: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9135939f-e992-427f-be1e-4ecfae2b319a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9135939f-e992-427f-be1e-4ecfae2b319a\",\n      \"name\": \"9135939f-e992-427f-be1e-4ecfae2b319a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:52:29.681806994Z\",\n      \"endTime\": \"2026-09-30T17:21:35.77427567Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-ec.1 is Partial (want Completed), started 22m59s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4e3ab6b49e6567cc40cafa154346d25ef5bfd7818598387865a31b0275872eb9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"alert","signature_id":"0d33f411900228bb6fdc790337157ac99d17d0d26ebbe8f2667d92df7f4ae88f","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T17:22:21Z\n  Ended:       2026-09-30T17:28:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"retina-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/services-j5446016\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/retina-agent has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T17:22:21Z\n  Ended:       2026-09-30T17:28:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"swift-recorder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/services-j5446016\", namespace=\"swift-recorder\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet swift-recorder/swift-recorder has not finished or progressed for at least 15 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-30T17:22:21Z\n  Ended:       2026-09-30T17:28:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"retina-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/services-j5446016\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/retina-agent has not finished or progressed for at least 15 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-30T17:22:21Z\n  Ended:       2026-09-30T17:28:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"set-kubelet-parameters-for-scale\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/services-j5446016\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/set-kubelet-parameters-for-scale has not finished or progressed for at least 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired"}},{"row_id":"50bdba9a945c69ff5ed2087ab5805766b9af93486dffbf3e9a30dacbe43e9e38","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"5120549ea0a4e43a54af415fcf4738f68bf5643f4b2148f738ff9eceeccfaf0b","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00113a0c0\u003e: \n    resource group \"private-kas-hm9pdnmdszt5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-hm9pdnmdszt5\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2f588ad3-c1af-45bb-9548-096867a64bbf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2f588ad3-c1af-45bb-9548-096867a64bbf\",\n      \"name\": \"2f588ad3-c1af-45bb-9548-096867a64bbf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:29:27.633769413Z\",\n      \"endTime\": \"2026-09-30T17:53:32.876325702Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f0n6rovpqtd5op2d9vogdgft7tah1 still exists (deletion dispatched at 2026-09-30T17:29:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00113a0c0\u003e: \n    resource group \"private-kas-hm9pdnmdszt5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-hm9pdnmdszt5\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2f588ad3-c1af-45bb-9548-096867a64bbf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2f588ad3-c1af-45bb-9548-096867a64bbf\",\n      \"name\": \"2f588ad3-c1af-45bb-9548-096867a64bbf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:29:27.633769413Z\",\n      \"endTime\": \"2026-09-30T17:53:32.876325702Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f0n6rovpqtd5op2d9vogdgft7tah1 still exists (deletion dispatched at 2026-09-30T17:29:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5a2f1c14b54bbf151c71f7611ce220c8f649fc58544dedc6741f9eb6cb082885","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"alert","signature_id":"8da3c2a194a0976325b473c920b87ca0f08f429d6596d56f440a5f88b7289803","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T17:22:35Z\n  Ended:       2026-09-30T17:57:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5446016-svc\", component=\"service-memory\", container=\"aro-hcp-frontend\", namespace=\"aro-hcp\", pod=\"aro-hcp-frontend-696bd4cfc8-xrrs2\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container aro-hcp-frontend in pod aro-hcp-frontend-696bd4cfc8-xrrs2 (namespace aro-hcp) on cluster ci01-j5446016-svc has averaged 170% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T17:22:35Z\n  Ended:       2026-09-30T18:09:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5446016-svc\", component=\"service-memory\", container=\"aro-hcp-frontend\", namespace=\"aro-hcp\", pod=\"aro-hcp-frontend-696bd4cfc8-bzvvh\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container aro-hcp-frontend in pod aro-hcp-frontend-696bd4cfc8-bzvvh (namespace aro-hcp) on cluster ci01-j5446016-svc has averaged 209.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"6f2a3d24dea55f39c32293802a0bd966db6157d8bba5efef9a125e32ff239396","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"76a0103093e7faf35353100b194b7ab9cbc8dcbb3627657d6f21d44e6293a1fc","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-622nv7gb94zh/cluster-candidate-4-22-jcthkg should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c422a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-jcthkg: failed waiting for cluster=\"cluster-candidate-4-22-jcthkg\" in resourcegroup=\"rg-candidate-4-22-622nv7gb94zh\" to finish creating: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c7ea5b6-e38d-4282-a8bd-b4e196c0a2b9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c7ea5b6-e38d-4282-a8bd-b4e196c0a2b9\",\n      \"name\": \"6c7ea5b6-e38d-4282-a8bd-b4e196c0a2b9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:52:38.429049015Z\",\n      \"endTime\": \"2026-09-30T17:21:45.984146934Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 22m41s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-622nv7gb94zh/cluster-candidate-4-22-jcthkg should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c422a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-jcthkg: failed waiting for cluster=\"cluster-candidate-4-22-jcthkg\" in resourcegroup=\"rg-candidate-4-22-622nv7gb94zh\" to finish creating: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c7ea5b6-e38d-4282-a8bd-b4e196c0a2b9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c7ea5b6-e38d-4282-a8bd-b4e196c0a2b9\",\n      \"name\": \"6c7ea5b6-e38d-4282-a8bd-b4e196c0a2b9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:52:38.429049015Z\",\n      \"endTime\": \"2026-09-30T17:21:45.984146934Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 22m41s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"77acf5c6a4c1ea1ce868ef21009e33b703b2ce98332fc81071291752cff27bbb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"fc6c0dff5250f75f2d3a3b3924e1b108f5097b463bf08d1d0cb2654087185cb1","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0029f60f0\u003e: \n    resource group \"complex-cilium-kv-vkmqwlbstl86\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-vkmqwlbstl86\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/758b0048-541b-41f7-b362-273a15532107\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/758b0048-541b-41f7-b362-273a15532107\",\n      \"name\": \"758b0048-541b-41f7-b362-273a15532107\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:24:38.640589117Z\",\n      \"endTime\": \"2026-09-30T17:48:45.86210432Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f1cult37rnvjnunif5m4d75jg68nc still exists (deletion dispatched at 2026-09-30T17:24:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0029f60f0\u003e: \n    resource group \"complex-cilium-kv-vkmqwlbstl86\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-vkmqwlbstl86\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/758b0048-541b-41f7-b362-273a15532107\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/758b0048-541b-41f7-b362-273a15532107\",\n      \"name\": \"758b0048-541b-41f7-b362-273a15532107\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:24:38.640589117Z\",\n      \"endTime\": \"2026-09-30T17:48:45.86210432Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f1cult37rnvjnunif5m4d75jg68nc still exists (deletion dispatched at 2026-09-30T17:24:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7ee7a67480aaf5942b4fa1b7932429433c85d22360ab642c189c1e18356ec6a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"alert","signature_id":"334ea0166a08010cff76f9d98df6bd32216ced385950ba9b5e9cd0a35c2140ea","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 93 time(s) (25 unknown, 68 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T17:04:15Z\n  Ended:       2026-09-30T17:27:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0m156ms43o6rvr33gnk6bjr3pfhe-a6a7t4q2j6v8c3t\", pod=\"etcd-2\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0m156ms43o6rvr33gnk6bjr3pfhe-a6a7t4q2j6v8c3t/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T17:05:13Z\n  Ended:       2026-09-30T17:27:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0e8dbaqbo8t5dt5vdcasi8k08a4q-v2v3z7b8u7q0n1w\", pod=\"kube-state-metrics-hcp-754f855b69-w9pxh\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0e8dbaqbo8t5dt5vdcasi8k08a4q-v2v3z7b8u7q0n1w/kube-state-metrics-hcp-754f855b69-w9pxh has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-30T17:05:13Z\n  Ended:       2026-09-30T17:34:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0lqhibjmokvvfkc1lvvbtckpvakc-y8w8x7w3j0c3f3n\", pod=\"router-79bddd4695-4xzx4\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0lqhibjmokvvfkc1lvvbtckpvakc-y8w8x7w3j0c3f3n/router-79bddd4695-4xzx4 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-30T17:05:13Z\n  Ended:       2026-09-30T17:27:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d\", pod=\"multus-admission-controller-77fb5c8d8f-hmg8g\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d/multus-admission-controller-77fb5c8d8f-hmg8g has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-30T17:05:13Z\n  Ended:       2026-09-30T17:34:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0jkpt55pcfu9tg9un71gt3tsm5a6-k9g8q9y9s6m6e2p\", pod=\"router-6c4c4f9b46-9jc7h\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0jkpt55pcfu9tg9un71gt3tsm5a6-k9g8q9y9s6m6e2p/router-6c4c4f9b46-9jc7h has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-30T17:05:13Z\n  Ended:       2026-09-30T17:27:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d\", pod=\"network-node-identity-7b6f464c54-ttl5f\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d/network-node-identity-7b6f464c54-ttl5f has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-09-30T17:05:13Z\n  Ended:       2026-09-30T17:27:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0lqugf7p62g2vu15647hciuo7rqe-ephemeral-disk\", pod=\"kube-state-metrics-hcp-754f855b69-5lx7w\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0lqugf7p62g2vu15647hciuo7rqe-ephemeral-disk/kube-state-metrics-hcp-754f855b69-5lx7w has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-09-30T17:05:13Z\n  Ended:       2026-09-30T17:27:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f1mqu517if66tcq7vv1883rm6svr2-x0m6x6k6h4c6v2j\", pod=\"kube-state-metrics-hcp-754f855b69-cvfnm\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f1mqu517if66tcq7vv1883rm6svr2-x0m6x6k6h4c6v2j/kube-state-metrics-hcp-754f855b69-cvfnm has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-09-30T17:05:13Z\n  Ended:       2026-09-30T17:27:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f19n96q5slf0m0lsru0ta7ea87dun-h5n4x4c7s1h6n9x\", pod=\"kube-state-metrics-hcp-754f855b69-5ds5m\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f19n96q5slf0m0lsru0ta7ea87dun-h5n4x4c7s1h6n9x/kube-state-metrics-hcp-754f855b69-5ds5m has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-09-30T17:05:13Z\n  Ended:       2026-09-30T17:27:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d\", pod=\"ovnkube-control-plane-549bdb54d-n62hb\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d/ovnkube-control-plane-549bdb54d-n62hb has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-09-30T17:06:13Z\n  Ended:       2026-09-30T17:27:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0g0m8v1rghh660hh9lh81tcp6q5n-private-ingress\", pod=\"network-node-identity-5469fd6495-rbqgx\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0g0m8v1rghh660hh9lh81tcp6q5n-private-ingress/network-node-identity-5469fd6495-rbqgx has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-09-30T17:06:14Z\n  Ended:       2026-09-30T17:27:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d\", pod=\"kube-controller-manager-59b5946b4c-8dnp4\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d/kube-controller-manager-59b5946b4c-8dnp4 has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-09-30T17:08:15Z\n  Ended:       2026-09-30T17:14:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0nnvckp9ok2ivvajoua5mi6rfcis-e4a2q0w1u4z3v6w\", pod=\"openshift-apiserver-5b6785f9dc-fc7tr\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0nnvckp9ok2ivvajoua5mi6rfcis-e4a2q0w1u4z3v6w/openshift-apiserver-5b6785f9dc-fc7tr has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n  State:       Resolved\n  Started:     2026-09-30T17:08:15Z\n  Ended:       2026-09-30T17:14:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f15npkrjd452rtt7ar07bisihfskg-j2c7h4h2c6p4s3p\", pod=\"openshift-apiserver-7dd55877cd-gkcd5\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f15npkrjd452rtt7ar07bisihfskg-j2c7h4h2c6p4s3p/openshift-apiserver-7dd55877cd-gkcd5 has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n  State:       Resolved\n  Started:     2026-09-30T17:08:15Z\n  Ended:       2026-09-30T17:14:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0uvlpj2ovpuf5b8grf3i9edvpodq-g3v6g1h5e0h5l2j\", pod=\"openshift-apiserver-758c4878d9-7j48d\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0uvlpj2ovpuf5b8grf3i9edvpodq-g3v6g1h5e0h5l2j/openshift-apiserver-758c4878d9-7j48d has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n  State:       Resolved\n  Started:     2026-09-30T17:11:13Z\n  Ended:       2026-09-30T17:27:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0g0m8v1rghh660hh9lh81tcp6q5n-private-ingress\", pod=\"network-node-identity-5469fd6495-f7zgd\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0g0m8v1rghh660hh9lh81tcp6q5n-private-ingress/network-node-identity-5469fd6495-f7zgd has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n  State:       Resolved\n  Started:     2026-09-30T17:11:13Z\n  Ended:       2026-09-30T17:27:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0m156ms43o6rvr33gnk6bjr3pfhe-a6a7t4q2j6v8c3t\", pod=\"router-8565cb6474-894zb\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0m156ms43o6rvr33gnk6bjr3pfhe-a6a7t4q2j6v8c3t/router-8565cb6474-894zb has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n  State:       Resolved\n  Started:     2026-09-30T17:11:13Z\n  Ended:       2026-09-30T17:27:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d\", pod=\"kube-apiserver-674f58879-j82fb\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d/kube-apiserver-674f58879-j82fb has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n  State:       Resolved\n  Started:     2026-09-30T17:11:14Z\n  Ended:       2026-09-30T17:27:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d\", pod=\"network-node-identity-7b6f464c54-gjfm6\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d/network-node-identity-7b6f464c54-gjfm6 has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n  State:       Resolved\n  Started:     2026-09-30T17:11:14Z\n  Ended:       2026-09-30T17:27:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d\", pod=\"openshift-apiserver-f89c544f9-jcfcs\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d/openshift-apiserver-f89c544f9-jcfcs has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n  State:       Resolved\n  Started:     2026-09-30T17:11:14Z\n  Ended:       2026-09-30T17:27:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0suq7elir5mh0aa877skrjk5onf2-t7t6k6q0r6g9t5v\", pod=\"openshift-apiserver-5d4fc6f8c7-68m94\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0suq7elir5mh0aa877skrjk5onf2-t7t6k6q0r6g9t5v/openshift-apiserver-5d4fc6f8c7-68m94 has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n  State:       Resolved\n  Started:     2026-09-30T17:11:14Z\n  Ended:       2026-09-30T17:27:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0suq7elir5mh0aa877skrjk5onf2-t7t6k6q0r6g9t5v\", pod=\"kube-apiserver-87c8b5fd9-ph74m\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0suq7elir5mh0aa877skrjk5onf2-t7t6k6q0r6g9t5v/kube-apiserver-87c8b5fd9-ph74m has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n  State:       Resolved\n  Started:     2026-09-30T17:13:16Z\n  Ended:       2026-09-30T17:22:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-p75kr\", region=\"westus3\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-p75kr has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:13:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-xfhsn\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/ama-metrics-node-xfhsn has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:13:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"set-kubelet-parameters-for-scale-vdkwr\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/set-kubelet-parameters-for-scale-vdkwr has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"817d70175a946670d824df2994758e55662874cbe6bffc0f86e9fdb75bdb8136","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"78aa4ea7e4450ab671d1d7955078133f07d134b1528dcc6a4a47bb2d342a0b2e","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010c0210\u003e: \n    resource group \"private-keyvault-9g9ksdvdr2w9\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-9g9ksdvdr2w9\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/135095ef-4130-48de-a890-36a3f2ec4ba2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/135095ef-4130-48de-a890-36a3f2ec4ba2\",\n      \"name\": \"135095ef-4130-48de-a890-36a3f2ec4ba2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:18:16.6672625Z\",\n      \"endTime\": \"2026-09-30T17:42:26.002126676Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f1cjpvuq75dcn2gn08v3gqss4ksup still exists (deletion dispatched at 2026-09-30T17:18:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010c0210\u003e: \n    resource group \"private-keyvault-9g9ksdvdr2w9\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-9g9ksdvdr2w9\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/135095ef-4130-48de-a890-36a3f2ec4ba2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/135095ef-4130-48de-a890-36a3f2ec4ba2\",\n      \"name\": \"135095ef-4130-48de-a890-36a3f2ec4ba2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:18:16.6672625Z\",\n      \"endTime\": \"2026-09-30T17:42:26.002126676Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f1cjpvuq75dcn2gn08v3gqss4ksup still exists (deletion dispatched at 2026-09-30T17:18:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a6299742b1ff6c5e532899640f78e385eadf28d949a1e95dfc697f72ee1a89ed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"5fe0d75300377f26dad14459efb5218ed6d1b660024fd855d7728e47a4e9c833","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-m98nfkv66wgc/cluster-candidate-4-23-2zm5lz should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013ea120\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-2zm5lz: failed waiting for cluster=\"cluster-candidate-4-23-2zm5lz\" in resourcegroup=\"rg-candidate-4-23-m98nfkv66wgc\" to finish creating: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e5bac24-0540-4cfd-8079-7530283faa59\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e5bac24-0540-4cfd-8079-7530283faa59\",\n      \"name\": \"7e5bac24-0540-4cfd-8079-7530283faa59\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:52:39.678276861Z\",\n      \"endTime\": \"2026-09-30T17:21:45.946479348Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-m98nfkv66wgc/cluster-candidate-4-23-2zm5lz should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013ea120\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-2zm5lz: failed waiting for cluster=\"cluster-candidate-4-23-2zm5lz\" in resourcegroup=\"rg-candidate-4-23-m98nfkv66wgc\" to finish creating: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e5bac24-0540-4cfd-8079-7530283faa59\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e5bac24-0540-4cfd-8079-7530283faa59\",\n      \"name\": \"7e5bac24-0540-4cfd-8079-7530283faa59\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:52:39.678276861Z\",\n      \"endTime\": \"2026-09-30T17:21:45.946479348Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ab90cece596927f29bb9d77cee68680b6565fc35bcb0bb0f4102a61da6891b98","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"alert","signature_id":"77cae9b2004bd675a604ffcac759cea6f79b1231e19f28fc49345d31d320e06a","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 12 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:04:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/hcps-j5446016\", namespace=\"ocm-arohcpci01-2t5f0m156ms43o6rvr33gnk6bjr3pfhe-a6a7t4q2j6v8c3t\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5f0m156ms43o6rvr33gnk6bjr3pfhe-a6a7t4q2j6v8c3t/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:04:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/hcps-j5446016\", namespace=\"ocm-arohcpci01-2t5f0m156ms43o6rvr33gnk6bjr3pfhe-a6a7t4q2j6v8c3t\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5f0m156ms43o6rvr33gnk6bjr3pfhe-a6a7t4q2j6v8c3t/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:04:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/hcps-j5446016\", namespace=\"ocm-arohcpci01-2t5f0m156ms43o6rvr33gnk6bjr3pfhe-a6a7t4q2j6v8c3t\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5f0m156ms43o6rvr33gnk6bjr3pfhe-a6a7t4q2j6v8c3t/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:04:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/hcps-j5446016\", namespace=\"ocm-arohcpci01-2t5f0jkpt55pcfu9tg9un71gt3tsm5a6-k9g8q9y9s6m6e2p\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5f0jkpt55pcfu9tg9un71gt3tsm5a6-k9g8q9y9s6m6e2p/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:04:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/hcps-j5446016\", namespace=\"ocm-arohcpci01-2t5f0jkpt55pcfu9tg9un71gt3tsm5a6-k9g8q9y9s6m6e2p\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5f0jkpt55pcfu9tg9un71gt3tsm5a6-k9g8q9y9s6m6e2p/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:04:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/hcps-j5446016\", namespace=\"ocm-arohcpci01-2t5f0m156ms43o6rvr33gnk6bjr3pfhe-a6a7t4q2j6v8c3t\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5f0m156ms43o6rvr33gnk6bjr3pfhe-a6a7t4q2j6v8c3t/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:04:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/hcps-j5446016\", namespace=\"ocm-arohcpci01-2t5f0lqhibjmokvvfkc1lvvbtckpvakc-y8w8x7w3j0c3f3n\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5f0lqhibjmokvvfkc1lvvbtckpvakc-y8w8x7w3j0c3f3n/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:04:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/hcps-j5446016\", namespace=\"ocm-arohcpci01-2t5f0lqhibjmokvvfkc1lvvbtckpvakc-y8w8x7w3j0c3f3n\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5f0lqhibjmokvvfkc1lvvbtckpvakc-y8w8x7w3j0c3f3n/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:04:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/hcps-j5446016\", namespace=\"ocm-arohcpci01-2t5f0lqhibjmokvvfkc1lvvbtckpvakc-y8w8x7w3j0c3f3n\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5f0lqhibjmokvvfkc1lvvbtckpvakc-y8w8x7w3j0c3f3n/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:04:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/hcps-j5446016\", namespace=\"ocm-arohcpci01-2t5f0lqhibjmokvvfkc1lvvbtckpvakc-y8w8x7w3j0c3f3n\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5f0lqhibjmokvvfkc1lvvbtckpvakc-y8w8x7w3j0c3f3n/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:04:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/hcps-j5446016\", namespace=\"ocm-arohcpci01-2t5f0jkpt55pcfu9tg9un71gt3tsm5a6-k9g8q9y9s6m6e2p\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5f0jkpt55pcfu9tg9un71gt3tsm5a6-k9g8q9y9s6m6e2p/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:04:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/hcps-j5446016\", namespace=\"ocm-arohcpci01-2t5f0jkpt55pcfu9tg9un71gt3tsm5a6-k9g8q9y9s6m6e2p\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5f0jkpt55pcfu9tg9un71gt3tsm5a6-k9g8q9y9s6m6e2p/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"b1c51a936db26863920a8afdc1896743fd6da2b5b06d9d7623fc4d6469e5d2b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"6616b57a77390d9fcea017a145f401ec9a44bec8381030ff7da7d5b66342bea0","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0029c4180\u003e: \n    resource group \"feature-aggregation-bffzrcs9xcvk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-bffzrcs9xcvk\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b2db4f57-9d20-454f-8d38-b79b992f2f32\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b2db4f57-9d20-454f-8d38-b79b992f2f32\",\n      \"name\": \"b2db4f57-9d20-454f-8d38-b79b992f2f32\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:34:12.215869805Z\",\n      \"endTime\": \"2026-09-30T17:58:12.821045662Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f1rm3k8s9324kfkruicq09u1qjvn0 still exists (deletion dispatched at 2026-09-30T17:34:14Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0029c4180\u003e: \n    resource group \"feature-aggregation-bffzrcs9xcvk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-bffzrcs9xcvk\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b2db4f57-9d20-454f-8d38-b79b992f2f32\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b2db4f57-9d20-454f-8d38-b79b992f2f32\",\n      \"name\": \"b2db4f57-9d20-454f-8d38-b79b992f2f32\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:34:12.215869805Z\",\n      \"endTime\": \"2026-09-30T17:58:12.821045662Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f1rm3k8s9324kfkruicq09u1qjvn0 still exists (deletion dispatched at 2026-09-30T17:34:14Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"dec046171ebe607223e3946d8ec1e31bae7db85e40dc9cfdf5b3ea1927a673f1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"c47e9e92a6f4e295102296fad094562a1d04cf7aee29b799aa62c27f9514ae5c","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1a380\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-rwh7s9rvp89r\" to finish creating: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb690e90-f4f6-4eb4-9d4f-fe046ccd3d44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb690e90-f4f6-4eb4-9d4f-fe046ccd3d44\",\n      \"name\": \"eb690e90-f4f6-4eb4-9d4f-fe046ccd3d44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:52:15.764760069Z\",\n      \"endTime\": \"2026-09-30T17:21:16.086415783Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 24m59s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1a380\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-rwh7s9rvp89r\" to finish creating: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb690e90-f4f6-4eb4-9d4f-fe046ccd3d44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb690e90-f4f6-4eb4-9d4f-fe046ccd3d44\",\n      \"name\": \"eb690e90-f4f6-4eb4-9d4f-fe046ccd3d44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:52:15.764760069Z\",\n      \"endTime\": \"2026-09-30T17:21:16.086415783Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 24m59s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e8724405a76b7347f766f06c463ec97e9d46716b0d7dc84ffa79d86931570487","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"9ebe0a8379936d0120b8baffd7195e8f84ec0ac55ddd2ce57a2b08da074be15e","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0001aacc0\u003e: \n    resource group \"rg-hs-pre--r48mqt9lrw4s\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-fqsxq9\" in resourcegroup=\"rg-hs-pre--r48mqt9lrw4s\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49c44789-52a8-49c2-b053-4835c3571170\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49c44789-52a8-49c2-b053-4835c3571170\",\n      \"name\": \"49c44789-52a8-49c2-b053-4835c3571170\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:16:19.615591526Z\",\n      \"endTime\": \"2026-09-30T17:40:25.981909891Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f0hjlm2j48ooq92uj0su5rbnsq03t still exists (deletion dispatched at 2026-09-30T17:16:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0001aacc0\u003e: \n    resource group \"rg-hs-pre--r48mqt9lrw4s\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-fqsxq9\" in resourcegroup=\"rg-hs-pre--r48mqt9lrw4s\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49c44789-52a8-49c2-b053-4835c3571170\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49c44789-52a8-49c2-b053-4835c3571170\",\n      \"name\": \"49c44789-52a8-49c2-b053-4835c3571170\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:16:19.615591526Z\",\n      \"endTime\": \"2026-09-30T17:40:25.981909891Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f0hjlm2j48ooq92uj0su5rbnsq03t still exists (deletion dispatched at 2026-09-30T17:16:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f315cd50d7b3b94dd0b6a695aef35c48e56ac58791ce91bcaebd9aebff7e3076","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"335ab959219cb206e30234fc1cf72ca10d7c54d0f8706508b7269b3ad089b86d","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010a8280\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-ztrwgb4tgx4r\" to finish creating: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fb9dfc1a-3b4b-487f-b91c-0f6b1985bf0d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fb9dfc1a-3b4b-487f-b91c-0f6b1985bf0d\",\n      \"name\": \"fb9dfc1a-3b4b-487f-b91c-0f6b1985bf0d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:52:39.354643704Z\",\n      \"endTime\": \"2026-09-30T17:21:45.995494198Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 25m45s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010a8280\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-ztrwgb4tgx4r\" to finish creating: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fb9dfc1a-3b4b-487f-b91c-0f6b1985bf0d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fb9dfc1a-3b4b-487f-b91c-0f6b1985bf0d\",\n      \"name\": \"fb9dfc1a-3b4b-487f-b91c-0f6b1985bf0d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:52:39.354643704Z\",\n      \"endTime\": \"2026-09-30T17:21:45.995494198Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 25m45s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":23,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","519fe0fe7478bebfb7ef939c84f76fa0281886d85cb376148838a1878147b618","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","c65eaecf2bcf9cab816e9e9171a4f334676466d8dfb2f88bb604984fc8e6cfe2","d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","e4f1a1ea46940043a3767809b952d9be40a221815d3f81ceb038491c52599831","f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c"],"matched_occurrences":23,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7254/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105322513406889984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7254,"pr_state":"closed","pr_sha":"fed46bdb633e2219c0b9e04a9984e1a83a49fd45","final_merged_sha":"fed46bdb633e2219c0b9e04a9984e1a83a49fd45","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T15:42:56Z","started_at":"2026-09-30T15:42:56Z","completed_at":"2026-09-30T18:13:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T18:29:07.618773939Z","timing_metadata_checked_at":"2026-09-30T18:29:07.618773939Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T18:29:07.621380532Z","region_metadata_checked_at":"2026-09-30T18:29:07.621380532Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7242/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105320779594862592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7242,"pr_state":"closed","pr_sha":"136c196e662abb71c58ecca8b6794334296a7ffd","final_merged_sha":"136c196e662abb71c58ecca8b6794334296a7ffd","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T15:36:03Z","started_at":"2026-09-30T15:36:03Z","completed_at":"2026-09-30T18:00:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T18:08:58.133101876Z","timing_metadata_checked_at":"2026-09-30T18:08:58.133101876Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T18:08:58.135892768Z","region_metadata_checked_at":"2026-09-30T18:08:58.135892768Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"6e4cb3f085907f3c3864d931651ab5669b26a959dec6014f8a1ddf72ce999234","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7242/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105320779594862592","occurred_at":"2026-09-30T15:36:03Z","failed_at":"e2e","signature_id":"c1bac4e92db70f51e3a0c22de77dcee2f5a3f262e7aa5d1b8931dc415680d6be","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ca3800\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-xw6tpr5d2lsl\" to finish creating: GET https://rp.j4862592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3d2602fc-b3fa-43e1-b8e3-5be7f209f5fc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3d2602fc-b3fa-43e1-b8e3-5be7f209f5fc\",\n      \"name\": \"3d2602fc-b3fa-43e1-b8e3-5be7f209f5fc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:03:28.253892004Z\",\n      \"endTime\": \"2026-09-30T17:22:36.043787859Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine oidc-wi-cluster-oidc-wi-np-h2g79-bb4np: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-xw6tpr5d2lsl--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-h2g79-bb4np to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine oidc-wi-cluster-oidc-wi-np-h2g79-bb4np: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-xw6tpr5d2lsl--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-h2g79-bb4np to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ca3800\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-xw6tpr5d2lsl\" to finish creating: GET https://rp.j4862592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3d2602fc-b3fa-43e1-b8e3-5be7f209f5fc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3d2602fc-b3fa-43e1-b8e3-5be7f209f5fc\",\n      \"name\": \"3d2602fc-b3fa-43e1-b8e3-5be7f209f5fc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:03:28.253892004Z\",\n      \"endTime\": \"2026-09-30T17:22:36.043787859Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine oidc-wi-cluster-oidc-wi-np-h2g79-bb4np: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-xw6tpr5d2lsl--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-h2g79-bb4np to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine oidc-wi-cluster-oidc-wi-np-h2g79-bb4np: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-xw6tpr5d2lsl--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-h2g79-bb4np to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105320628276957184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6577,"pr_state":"closed","pr_sha":"8719a8a13bd69122b6b8841f4e14e467b41e9993","final_merged_sha":"3d03a40d19bf9b85b85165f4f9432cbbcf37480e","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T15:35:27Z","started_at":"2026-09-30T15:35:27Z","completed_at":"2026-09-30T18:05:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T18:08:55.716111282Z","timing_metadata_checked_at":"2026-09-30T18:08:55.716111282Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T18:08:55.719478573Z","region_metadata_checked_at":"2026-09-30T18:08:55.719478573Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6967/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105320296662700032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6967,"pr_state":"closed","pr_sha":"3a81d064b855c5a0862d9de651db3428e2519ace","final_merged_sha":"3a81d064b855c5a0862d9de651db3428e2519ace","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T15:34:08Z","started_at":"2026-09-30T15:34:08Z","completed_at":"2026-09-30T17:58:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T18:08:57.124022374Z","timing_metadata_checked_at":"2026-09-30T18:08:57.124022374Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T18:08:57.126950066Z","region_metadata_checked_at":"2026-09-30T18:08:57.126950066Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105310240105828352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7261,"pr_state":"closed","pr_sha":"e01d7004aa224ff873a1a813ac3e6932792e1135","final_merged_sha":"f6667070546b139bd53a7df7d7674fd0c4250636","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T14:54:10Z","started_at":"2026-09-30T14:54:10Z","completed_at":"2026-09-30T17:24:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T17:36:59.989528706Z","timing_metadata_checked_at":"2026-09-30T17:36:59.989528706Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T17:36:59.992520399Z","region_metadata_checked_at":"2026-09-30T17:36:59.992520399Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"8e7c8ad5f90ec5a260a9d8f0dd5acbd6718f58b843a38edcfddaffb9624db754","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105310240105828352","occurred_at":"2026-09-30T14:54:10Z","failed_at":"alert","signature_id":"14d730d688feed9879dfb08b05aec4624d9b99fa53a65137ea56efd900ad0e9d","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T16:05:30Z\n  Ended:       2026-09-30T16:33:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5828352-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5e65ks30us7u7a3qg1vdbknb1vvqs6-t3z0s0t0v4e0p5i\", pod=\"router-865f6cd877-rnq4k\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5e65ks30us7u7a3qg1vdbknb1vvqs6-t3z0s0t0v4e0p5i/router-865f6cd877-rnq4k has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T16:07:32Z\n  Ended:       2026-09-30T16:25:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5828352-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5e65gj37b763s4sdv3kievh5fge8ub-c7d3y8f1x8e6t7o\", pod=\"router-8f95b5bf6-pw2hq\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5e65gj37b763s4sdv3kievh5fge8ub-c7d3y8f1x8e6t7o/router-8f95b5bf6-pw2hq has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"a5901e8e124043896aab06ceb2f5b4dcffc9346624ce0b8afa494bbc23edb05e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105310240105828352","occurred_at":"2026-09-30T14:54:10Z","failed_at":"e2e","signature_id":"55c7211b6b6d239408f79dea1df4f8eea0fbc4889550177ece4353034cb083f3","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-xnh68bcrp77f/cluster-candidate-4-21-944b8p should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000874f60\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-944b8p: failed waiting for cluster=\"cluster-candidate-4-21-944b8p\" in resourcegroup=\"rg-candidate-4-21-xnh68bcrp77f\" to finish creating: GET https://rp.j5828352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0796d72b-193f-46f8-9e78-aaf27448038b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0796d72b-193f-46f8-9e78-aaf27448038b\",\n      \"name\": \"0796d72b-193f-46f8-9e78-aaf27448038b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T15:56:05.509326654Z\",\n      \"endTime\": \"2026-09-30T16:25:07.850265278Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 25m16s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-xnh68bcrp77f/cluster-candidate-4-21-944b8p should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000874f60\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-944b8p: failed waiting for cluster=\"cluster-candidate-4-21-944b8p\" in resourcegroup=\"rg-candidate-4-21-xnh68bcrp77f\" to finish creating: GET https://rp.j5828352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0796d72b-193f-46f8-9e78-aaf27448038b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0796d72b-193f-46f8-9e78-aaf27448038b\",\n      \"name\": \"0796d72b-193f-46f8-9e78-aaf27448038b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T15:56:05.509326654Z\",\n      \"endTime\": \"2026-09-30T16:25:07.850265278Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 25m16s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105308204090003456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7143,"pr_state":"open","pr_sha":"cdf5413a948fe5306876c42a99a496f2a0c8e352","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T14:46:05Z","started_at":"2026-09-30T14:46:05Z","completed_at":"2026-09-30T17:18:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T17:36:59.960296675Z","timing_metadata_checked_at":"2026-09-30T17:36:59.960296675Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T17:36:59.966010562Z","region_metadata_checked_at":"2026-09-30T17:36:59.966010562Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105298021888561152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6900,"pr_state":"closed","pr_sha":"7551c0170c5cffea9e64f3707dcf2378a40b7d3f","final_merged_sha":"d63de0c62ddb9d7d7de369b28c8d93807d13672c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T14:05:37Z","started_at":"2026-09-30T14:05:37Z","completed_at":"2026-09-30T16:47:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T17:05:27.830030213Z","timing_metadata_checked_at":"2026-09-30T17:05:27.830030213Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T17:05:27.8340818Z","region_metadata_checked_at":"2026-09-30T17:05:27.8340818Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"b8ebdf0940900016061394ed2ad755d1660f7cee2cebb7def0a9cfdda9f40918","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105298021888561152","occurred_at":"2026-09-30T14:05:37Z","failed_at":"alert","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T15:25:46Z\n  Ended:       2026-09-30T15:31:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8561152-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105297136064139264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7141,"pr_state":"closed","pr_sha":"6afba8239efe6245441f25e86ccd2ab7ba7f5035","final_merged_sha":"6afba8239efe6245441f25e86ccd2ab7ba7f5035","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T14:02:06Z","started_at":"2026-09-30T14:02:06Z","completed_at":"2026-09-30T16:47:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T17:05:28.238065946Z","timing_metadata_checked_at":"2026-09-30T17:05:28.238065946Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T17:05:28.241049837Z","region_metadata_checked_at":"2026-09-30T17:05:28.241049837Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"9fd7cfdd1f1fe96b8503a67f56e20510aff7491ca9c6c32c89349b13e11c8b24","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105297136064139264","occurred_at":"2026-09-30T14:02:06Z","failed_at":"alert","signature_id":"632e48363b16662cf206e035c9d5483425dd46982f9be6da0c90dbb11133e9ff","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T15:21:57Z\n  Ended:       2026-09-30T15:50:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4139264-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5dgduvrdd5glf5k0m6vesv46nt0jhm-private-kas\", pod=\"router-55bfdfc49b-qhf45\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5dgduvrdd5glf5k0m6vesv46nt0jhm-private-kas/router-55bfdfc49b-qhf45 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"be46e56c2bfaa5f6ebe5edb62bcb1125fffb033614a679ece26327f52a7d6553","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105297136064139264","occurred_at":"2026-09-30T14:02:06Z","failed_at":"e2e","signature_id":"5cbfd891b17369ec915b5a99867c4b2f1c6fc13517bd9b12c29a9682a6a58e0a","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001f61c0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-2ggb82wh5r22\" to finish creating: GET https://rp.j4139264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/50346f69-92e6-45cd-a218-e9b1ff541e03\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/50346f69-92e6-45cd-a218-e9b1ff541e03\",\n      \"name\": \"50346f69-92e6-45cd-a218-e9b1ff541e03\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T15:09:42.512906803Z\",\n      \"endTime\": \"2026-09-30T15:38:51.371261651Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 23m14s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001f61c0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-2ggb82wh5r22\" to finish creating: GET https://rp.j4139264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/50346f69-92e6-45cd-a218-e9b1ff541e03\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/50346f69-92e6-45cd-a218-e9b1ff541e03\",\n      \"name\": \"50346f69-92e6-45cd-a218-e9b1ff541e03\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T15:09:42.512906803Z\",\n      \"endTime\": \"2026-09-30T15:38:51.371261651Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 23m14s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6944/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105295109229645824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6944,"pr_state":"open","pr_sha":"945b5ce92c4d576dbad252aa980a2a25b088b764","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T13:54:03Z","started_at":"2026-09-30T13:54:03Z","completed_at":"2026-09-30T16:20:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T16:33:52.24692486Z","timing_metadata_checked_at":"2026-09-30T16:33:52.24692486Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T16:33:52.250565749Z","region_metadata_checked_at":"2026-09-30T16:33:52.250565749Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"0136923f00aa5803e69549b2f7c0b328d8673edf7206e3f483ddd8c0275b8653","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6944/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105295109229645824","occurred_at":"2026-09-30T13:54:03Z","failed_at":"alert","signature_id":"ae184cd60d96c8302bc4346af9b111ff0bed8c1839b502ea23adb68311ddbd47","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 7 time(s) (1 unknown, 6 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T15:24:36Z\n  Ended:       2026-09-30T15:30:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9645824-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5d7bu25qfk2cfhnp3n9qlq37dgcjbd-private-ingress\", pod=\"etcd-0\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5d7bu25qfk2cfhnp3n9qlq37dgcjbd-private-ingress/etcd-0 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"e0056126b95491bf1abddd1ed55a47da671b413bb44e9883287392b1bf030a8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6944/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105295109229645824","occurred_at":"2026-09-30T13:54:03Z","failed_at":"e2e","signature_id":"842ff61370b596e7c3d23fa920da25dad023b70b10d4ba2b38239c18c65c13b1","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:251]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0xc000c9ee28\u003e: \n    context deadline exceeded\n    failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:251]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0xc000c9ee28\u003e: \n    context deadline exceeded\n    failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105294563651358720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6820,"pr_state":"closed","pr_sha":"c99d95a578f89e3e4c3bfc476a031bb4b59056f4","final_merged_sha":"273677ce69aa7a8dc7e1da98b8942d3ea48b24d1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T13:51:53Z","started_at":"2026-09-30T13:51:53Z","completed_at":"2026-09-30T15:13:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T15:30:36.77474978Z","timing_metadata_checked_at":"2026-09-30T15:30:36.77474978Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T15:30:36.77796887Z","region_metadata_checked_at":"2026-09-30T15:30:36.77796887Z"},"failed_at":["alert","provision"],"failed_test_count":4,"occurrences":[{"row_id":"2f1dd9c1d7339dfb953eaed1a62cc085d124194a724ea931403eca39d85fab87","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105294563651358720","occurred_at":"2026-09-30T13:51:53Z","failed_at":"provision","signature_id":"116a812adf4da3f0b766ca7f0e970c4b7b28de7fba2c7153f753459f2b443b8c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","test_suite":"templatize-pipeline","failure_text":"time=2026-09-30T14:35:13.661Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-30T14:35:14.082Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j1358720-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:eb3e2e068172121a9ca133c5d17784df770dacc104f20c7d4a7df5f1c70e8a08 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:450m memory:472Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j1358720-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5c29681bb4b38d08ca21c31c4553bab72e8e9c260dc0ec875e1d014268a9e580 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:e962c7b1-5eb5-497b-9341-4b3a121f354e region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#e962c7b1-5eb5-497b-9341-4b3a121f354e INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-30T14:35:14.083Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T14:35:14.275Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-30T14:35:14.491Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-30T14:35:14.491Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T14:35:14.524Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T14:45:17.368Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T14:45:17.369Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T14:46:17.370Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-30T14:46:17.815Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j1358720-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:eb3e2e068172121a9ca133c5d17784df770dacc104f20c7d4a7df5f1c70e8a08 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:450m memory:472Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j1358720-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5c29681bb4b38d08ca21c31c4553bab72e8e9c260dc0ec875e1d014268a9e580 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:e962c7b1-5eb5-497b-9341-4b3a121f354e region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#e962c7b1-5eb5-497b-9341-4b3a121f354e INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-30T14:46:17.815Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T14:46:18.042Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-30T14:46:18.350Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T14:46:18.350Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T14:46:18.601Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-30T14:46:19.301Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T14:46:19.301Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T14:46:19.487Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T14:46:19.912Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T14:46:20.506Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T14:46:20.869Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T14:46:21.189Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T14:46:21.550Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T14:46:21.905Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-30T14:46:21.905Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T14:46:21.951Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-30T14:56:23.257Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T14:56:23.257Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T14:57:23.259Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-30T14:57:24.108Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j1358720-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:eb3e2e068172121a9ca133c5d17784df770dacc104f20c7d4a7df5f1c70e8a08 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:450m memory:472Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j1358720-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5c29681bb4b38d08ca21c31c4553bab72e8e9c260dc0ec875e1d014268a9e580 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:e962c7b1-5eb5-497b-9341-4b3a121f354e region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#e962c7b1-5eb5-497b-9341-4b3a121f354e INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-30T14:57:24.108Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T14:57:24.411Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-30T14:57:24.749Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T14:57:24.749Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T14:57:25.002Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-30T14:57:25.620Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T14:57:25.620Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T14:57:25.977Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T14:57:26.226Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T14:57:26.765Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T14:57:26.974Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T14:57:27.201Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T14:57:27.763Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T14:57:28.017Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-30T14:57:28.017Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T14:57:28.079Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-30T15:05:13.695Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"context deadline exceeded\"\ntime=2026-09-30T15:05:13.695Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","failure_pattern":"failed to roll out Helm release: context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded"}},{"row_id":"a3c608f908175e627dcf21ec2ce22918017d607cbfdd8561892810dabaf06d9f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105294563651358720","occurred_at":"2026-09-30T13:51:53Z","failed_at":"alert","signature_id":"6ab116244e8b6c55af7310ca393f986f1be56582f51b8e5b67d25e76cd3995b9","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T14:52:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j1358720-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.42:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1358720/providers/microsoft.monitor/accounts/services-j1358720\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-r6958\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T14:52:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j1358720-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.42:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1358720/providers/microsoft.monitor/accounts/services-j1358720\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-r6958\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired"}},{"row_id":"b044b6e7944e4c2d0ee8033dfbfc75317825839f2636b750e6bfd3db8dd63219","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105294563651358720","occurred_at":"2026-09-30T13:51:53Z","failed_at":"alert","signature_id":"b25416717b76880c6609e81cf02b1d01fc79601f025ad76a9d31a41cb5607615","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T14:42:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1358720-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-png7s\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-png7s has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T14:43:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1358720-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-bzbm7\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-bzbm7 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T14:46:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1358720-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-mkkjw\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-mkkjw has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"e472ccfbd946897ea421439e8a17caab6fd959a80bbedf5930c0ed20d8216866","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105294563651358720","occurred_at":"2026-09-30T13:51:53Z","failed_at":"alert","signature_id":"8d542cbb8d9ed9afab5c0b08d7fb2e462252dfaa5e9d052f0e6a6910d7b32fe6","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T14:51:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j1358720-svc\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"warning\"\n  Description: The Arobit forwarder metrics endpoint on cluster ci01-j1358720-svc has been unreachable for 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105294515421057024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7261,"pr_state":"closed","pr_sha":"eb16c2e94e7f2596a478e5570733fb66ffcca02f","final_merged_sha":"f6667070546b139bd53a7df7d7674fd0c4250636","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T13:51:41Z","started_at":"2026-09-30T13:51:41Z","completed_at":"2026-09-30T14:15:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T14:27:20.627331401Z","timing_metadata_checked_at":"2026-09-30T14:27:20.627331401Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-30T14:27:20.63036099Z","region_metadata_checked_at":"2026-09-30T15:30:38.135687941Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e2f618c09f9702822e92ca3e057c1fb271128da0e59eeb7a1a3ece0340469634","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105294515421057024","occurred_at":"2026-09-30T13:51:41Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105294443023175680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7112,"pr_state":"closed","pr_sha":"395e45d0bbde804a9c22a549e978e9371ff65d91","final_merged_sha":"c11d1aab795ebbd762ddef1d8188a105d875d002","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T13:51:24Z","started_at":"2026-09-30T13:51:24Z","completed_at":"2026-09-30T16:22:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T16:33:52.419994008Z","timing_metadata_checked_at":"2026-09-30T16:33:52.419994008Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T16:33:52.423155698Z","region_metadata_checked_at":"2026-09-30T16:33:52.423155698Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7269/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105294127649263616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7269,"pr_state":"closed","pr_sha":"ba33a85d4cb42b48ca7efbabd149d16d8e41c231","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T13:49:57Z","started_at":"2026-09-30T13:49:57Z","completed_at":"2026-09-30T16:11:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T16:23:15.939698075Z","timing_metadata_checked_at":"2026-09-30T16:23:15.939698075Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T16:23:15.942871665Z","region_metadata_checked_at":"2026-09-30T16:23:15.942871665Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"5188a67ac55429a9dadf31e3ef3b6bcc3649ff5d5b62e41fd496aed8ce8f6bf7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7269/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105294127649263616","occurred_at":"2026-09-30T13:49:57Z","failed_at":"e2e","signature_id":"04e00d32fdeba2bfdaf45c1fa9c7407287ff098c212502b818e7b2573e85be65","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:242]: failed to create HCP cluster with containerRegistry set\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b4a880\u003e: \n    failed starting cluster creation \"acr-pull-create\" in resourcegroup=\"acr-pull-create-v5r8zv9r2jvq\": PUT https://rp.j9263616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/acr-pull-create-v5r8zv9r2jvq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/acr-pull-create\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/acr-pull-create-v5r8zv9r2jvq/providers/Microsoft.ManagedIdentity/userAssignedIdentities/acr-pull-mi\\\": containerRegistry requires cluster version 4.22.0 or above\",\n        \"target\": \"properties.platform.containerRegistry.managedIdentity\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:242]: failed to create HCP cluster with containerRegistry set\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b4a880\u003e: \n    failed starting cluster creation \"acr-pull-create\" in resourcegroup=\"acr-pull-create-v5r8zv9r2jvq\": PUT https://rp.j9263616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/acr-pull-create-v5r8zv9r2jvq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/acr-pull-create\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/acr-pull-create-v5r8zv9r2jvq/providers/Microsoft.ManagedIdentity/userAssignedIdentities/acr-pull-mi\\\": containerRegistry requires cluster version 4.22.0 or above\",\n        \"target\": \"properties.platform.containerRegistry.managedIdentity\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e648505864d678d9f50fe37f530cf1b1dab08c26dec37f3ae5734914d2656ab9","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/acr-pull-mi\": containerRegistry requires cluster version 4.22.0 or above; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"7b4b85545a2724ae2c05a95fbaa529e63e48b14e3ffac9bd1cce64cc6bb215a5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7269/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105294127649263616","occurred_at":"2026-09-30T13:49:57Z","failed_at":"e2e","signature_id":"d6cef5f759144348562c7441abc3cf578b5ca595385f45cabf910f3ccade4227","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:96]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c48700\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-7mqkjzxbfvhm\": PUT https://rp.j9263616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-7mqkjzxbfvhm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"Private\\\": not supported when customerProperties.version.id is below 4.22\",\n        \"target\": \"properties.api.visibility\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:96]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c48700\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-7mqkjzxbfvhm\": PUT https://rp.j9263616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-7mqkjzxbfvhm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"Private\\\": not supported when customerProperties.version.id is below 4.22\",\n        \"target\": \"properties.api.visibility\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"082efb8c79c4bc42f53051748bec5dc917c7daff5a16459440cdcba2f63beaf8","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"Private\": not supported when customerProperties.version.id is below 4.22; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"f413f40c2294cef62269313dc811d51dd9567628cdb3c9503e66686227fa935c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7269/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105294127649263616","occurred_at":"2026-09-30T13:49:57Z","failed_at":"e2e","signature_id":"6693a3c38bba6bece1046d71a702c519b29e94b4f42ba132e9c2c5a8c666092f","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:191]: failed to update cluster with new KMS key\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000ece360\u003e: \n    PATCH https://rp.j9263616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kms-key-rotate-s7rqrhtvbcff/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/kms-key-rotate-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"484184ba23584b37813d670c444e552b\\\": KMS key version rotation requires cluster version 4.22.0 or above\",\n        \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.activeKey.version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:191]: failed to update cluster with new KMS key\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000ece360\u003e: \n    PATCH https://rp.j9263616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kms-key-rotate-s7rqrhtvbcff/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/kms-key-rotate-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"484184ba23584b37813d670c444e552b\\\": KMS key version rotation requires cluster version 4.22.0 or above\",\n        \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.activeKey.version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fdef26532b23d7ffbf4352a28e772bf3eb538e6b3ef837cff2ecfc32992615a2","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003chex\u003e\": KMS key version rotation requires cluster version 4.22.0 or above; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["082efb8c79c4bc42f53051748bec5dc917c7daff5a16459440cdcba2f63beaf8","e648505864d678d9f50fe37f530cf1b1dab08c26dec37f3ae5734914d2656ab9","fdef26532b23d7ffbf4352a28e772bf3eb538e6b3ef837cff2ecfc32992615a2"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7268/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105293545626669056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7268,"pr_state":"open","pr_sha":"035752c2d1ac4dfa83de437669f06c1bbd17fecb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T13:47:36Z","started_at":"2026-09-30T13:47:36Z","completed_at":"2026-09-30T16:19:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T16:33:52.645926887Z","timing_metadata_checked_at":"2026-09-30T16:33:52.645926887Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T16:33:52.651686868Z","region_metadata_checked_at":"2026-09-30T16:33:52.651686868Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7246/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105289648665268224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7246,"pr_state":"closed","pr_sha":"26be4d462d878ae28865cbe5d86888fee258acfb","final_merged_sha":"26be4d462d878ae28865cbe5d86888fee258acfb","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T13:32:21Z","started_at":"2026-09-30T13:32:21Z","completed_at":"2026-09-30T15:48:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T16:02:49.763342381Z","timing_metadata_checked_at":"2026-09-30T16:02:49.763342381Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T16:02:49.766475871Z","region_metadata_checked_at":"2026-09-30T16:02:49.766475871Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105279807200694272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7053,"pr_state":"closed","pr_sha":"8470e4a8152fec1ad8e98ecdb32250dd598096a9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T12:53:14Z","started_at":"2026-09-30T12:53:14Z","completed_at":"2026-09-30T15:14:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T15:30:38.051011611Z","timing_metadata_checked_at":"2026-09-30T15:30:38.051011611Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T15:30:38.0544819Z","region_metadata_checked_at":"2026-09-30T15:30:38.0544819Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"7a4c656e98cd11f37645f828fbef44146a64ed5a6d9807f5845edc915a3be75f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105279807200694272","occurred_at":"2026-09-30T12:53:14Z","failed_at":"e2e","signature_id":"d84fbb4f91a7ca89a512a9ccaac84a33ddc34058ff07b5aaf957fe0df53b120b","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ea00e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-7mqggzhqq55s\" to finish creating: GET https://rp.j0694272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ec169ee-a122-45a3-a622-b7083455fb90\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ec169ee-a122-45a3-a622-b7083455fb90\",\n      \"name\": \"1ec169ee-a122-45a3-a622-b7083455fb90\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T14:16:43.796681639Z\",\n      \"endTime\": \"2026-09-30T14:35:52.497968724Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y5o8a1j8f7l2m2y-np-1-qdcxj-hwbpg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y5o8a1j8f7l2m2y-np-1-qdcxj-hwbpg: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ea00e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-7mqggzhqq55s\" to finish creating: GET https://rp.j0694272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ec169ee-a122-45a3-a622-b7083455fb90\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ec169ee-a122-45a3-a622-b7083455fb90\",\n      \"name\": \"1ec169ee-a122-45a3-a622-b7083455fb90\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T14:16:43.796681639Z\",\n      \"endTime\": \"2026-09-30T14:35:52.497968724Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y5o8a1j8f7l2m2y-np-1-qdcxj-hwbpg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y5o8a1j8f7l2m2y-np-1-qdcxj-hwbpg: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fd0407baf94133db7fa1758b83668acc74fdd3ca69dbe3150c9481e216c634ef","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"fbcfa983645c3e2633f472d4edb029f4fb86ade620ae9a86f76fa5ddd9dc3a24","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105279807200694272","occurred_at":"2026-09-30T12:53:14Z","failed_at":"e2e","signature_id":"b67af1235e148ac25954a38fdd4d5236c882e2e12c2417baba9e32624257a663","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:600]: failed to create nodepool npdg-4-21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000686fa0\u003e: \n    failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-hqw2kg\" in resourcegroup=\"rg-np-downgrade-hqw2kg-tzshqkp8zjqk\" to finish creating: GET https://rp.j0694272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0c88c27e-4a0d-4fb4-a4af-b21df9d3ccdc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0c88c27e-4a0d-4fb4-a4af-b21df9d3ccdc\",\n      \"name\": \"0c88c27e-4a0d-4fb4-a4af-b21df9d3ccdc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T14:18:31.838214826Z\",\n      \"endTime\": \"2026-09-30T14:37:32.428641421Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine h2z3j3d0p8e5c3v-npdg-4-21-7x86w-jpqjj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-hqw2kg-tzshqkp8zjqk-np-dg-hqw2kg--managed/providers/Microsoft.Compute/virtualMachines/h2z3j3d0p8e5c3v-npdg-4-21-7x86w-jpqjj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine h2z3j3d0p8e5c3v-npdg-4-21-7x86w-jpqjj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-hqw2kg-tzshqkp8zjqk-np-dg-hqw2kg--managed/providers/Microsoft.Compute/virtualMachines/h2z3j3d0p8e5c3v-npdg-4-21-7x86w-jpqjj to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:600]: failed to create nodepool npdg-4-21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000686fa0\u003e: \n    failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-hqw2kg\" in resourcegroup=\"rg-np-downgrade-hqw2kg-tzshqkp8zjqk\" to finish creating: GET https://rp.j0694272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0c88c27e-4a0d-4fb4-a4af-b21df9d3ccdc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0c88c27e-4a0d-4fb4-a4af-b21df9d3ccdc\",\n      \"name\": \"0c88c27e-4a0d-4fb4-a4af-b21df9d3ccdc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T14:18:31.838214826Z\",\n      \"endTime\": \"2026-09-30T14:37:32.428641421Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine h2z3j3d0p8e5c3v-npdg-4-21-7x86w-jpqjj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-hqw2kg-tzshqkp8zjqk-np-dg-hqw2kg--managed/providers/Microsoft.Compute/virtualMachines/h2z3j3d0p8e5c3v-npdg-4-21-7x86w-jpqjj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine h2z3j3d0p8e5c3v-npdg-4-21-7x86w-jpqjj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-hqw2kg-tzshqkp8zjqk-np-dg-hqw2kg--managed/providers/Microsoft.Compute/virtualMachines/h2z3j3d0p8e5c3v-npdg-4-21-7x86w-jpqjj to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","fd0407baf94133db7fa1758b83668acc74fdd3ca69dbe3150c9481e216c634ef"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105270530377519104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6915,"pr_state":"open","pr_sha":"0bdd36fc9bed8d937676c8592b848cd9cbcde70d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T12:16:23Z","started_at":"2026-09-30T12:16:23Z","completed_at":"2026-09-30T14:47:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T14:58:38.59964585Z","timing_metadata_checked_at":"2026-09-30T14:58:38.59964585Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T14:58:38.60287854Z","region_metadata_checked_at":"2026-09-30T14:58:38.60287854Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7260/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105265973241057280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7260,"pr_state":"closed","pr_sha":"d2063f84c5105758d4ff2370d77bd0c238c14b4d","final_merged_sha":"e2029a06a51da6cdf2ed5b5ef1c620dcc9cda152","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T11:58:16Z","started_at":"2026-09-30T11:58:16Z","completed_at":"2026-09-30T14:35:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T14:48:04.941795603Z","timing_metadata_checked_at":"2026-09-30T14:48:04.941795603Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T14:48:04.944855194Z","region_metadata_checked_at":"2026-09-30T14:48:04.944855194Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7158/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105265916785725440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7158,"pr_state":"open","pr_sha":"0ca8c9c41a80a8dbda691ac0542184fa1ebc5996","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T11:58:03Z","started_at":"2026-09-30T11:58:03Z","completed_at":"2026-09-30T14:21:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T14:38:02.18671305Z","timing_metadata_checked_at":"2026-09-30T14:38:02.18671305Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T14:38:02.194917923Z","region_metadata_checked_at":"2026-09-30T14:38:02.194917923Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7263/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105257717114343424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7263,"pr_state":"closed","pr_sha":"8481983b584ac7681861ebe6510e693bba610634","final_merged_sha":"8481983b584ac7681861ebe6510e693bba610634","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T11:25:28Z","started_at":"2026-09-30T11:25:28Z","completed_at":"2026-09-30T13:43:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T13:55:37.501328185Z","timing_metadata_checked_at":"2026-09-30T13:55:37.501328185Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T13:55:37.504463077Z","region_metadata_checked_at":"2026-09-30T13:55:37.504463077Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105255814120869888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7250,"pr_state":"open","pr_sha":"834e3a4bfb7f68837a296014f2446af7c338f3b8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T11:17:54Z","started_at":"2026-09-30T11:17:54Z","completed_at":"2026-09-30T13:54:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T14:06:22.890288049Z","timing_metadata_checked_at":"2026-09-30T14:06:22.890288049Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T14:06:22.893614938Z","region_metadata_checked_at":"2026-09-30T14:06:22.893614938Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105253779644354560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6141,"pr_state":"open","pr_sha":"a0b9d9de5a50a5434f1e58fba07352436371588c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T11:09:49Z","started_at":"2026-09-30T11:09:49Z","completed_at":"2026-09-30T13:26:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T13:35:24.759114007Z","timing_metadata_checked_at":"2026-09-30T13:35:24.759114007Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T13:35:24.761907896Z","region_metadata_checked_at":"2026-09-30T13:35:24.761907896Z"},"failed_at":["e2e"],"failed_test_count":10,"occurrences":[{"row_id":"20384cbfa72afe9dff06179fe6fb87433c2c519fd027d233b59ba3773ca7f40e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105253779644354560","occurred_at":"2026-09-30T11:09:49Z","failed_at":"e2e","signature_id":"a7d987a6289e47bbd6f06053a8e44ff6a83f6ca7a82baaa9d31d2b45509ddaa3","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-889bmn\" for cluster \"cluster-candidate-4-20-889bmn\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015841e0\u003e: \n    failed to create NodePool np-889bmn: failed waiting for nodepool=\"np-889bmn\" for cluster \"cluster-candidate-4-20-889bmn\" in resourcegroup=\"rg-candidate-4-20-qcvff5h4wj94\" to finish creating: GET https://rp.j4354560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e30a66c5-dd1a-49f7-a04b-0cec3c58378a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e30a66c5-dd1a-49f7-a04b-0cec3c58378a\",\n      \"name\": \"e30a66c5-dd1a-49f7-a04b-0cec3c58378a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T12:19:11.883646471Z\",\n      \"endTime\": \"2026-09-30T12:38:11.728721311Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-889bmn\" for cluster \"cluster-candidate-4-20-889bmn\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015841e0\u003e: \n    failed to create NodePool np-889bmn: failed waiting for nodepool=\"np-889bmn\" for cluster \"cluster-candidate-4-20-889bmn\" in resourcegroup=\"rg-candidate-4-20-qcvff5h4wj94\" to finish creating: GET https://rp.j4354560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e30a66c5-dd1a-49f7-a04b-0cec3c58378a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e30a66c5-dd1a-49f7-a04b-0cec3c58378a\",\n      \"name\": \"e30a66c5-dd1a-49f7-a04b-0cec3c58378a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T12:19:11.883646471Z\",\n      \"endTime\": \"2026-09-30T12:38:11.728721311Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"64c2d1d9234c34a6135ae91e01124342e7ff5dd9dc88a4e82dbf77091dc128de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"25064b267613d8f703d629d9b485be5bba78d318d63590a4ace739fc27ee79e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105253779644354560","occurred_at":"2026-09-30T11:09:49Z","failed_at":"e2e","signature_id":"31f7686d857db3ec2fac93987fae6b6a3245782f4b94f1b84b4b75f60d2d2c13","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:615]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:615]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"97763ea227d764b6cace3c59c3e2291599eb63cdd34dd1463aad690bc17db216","failure_pattern":"expected node pool nodes to report at least one release image ref before downgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:615]: expected node pool nodes to report at least one release image ref before downgrade"}},{"row_id":"336db8d1f67cf62217ce7d703879cf57b593810d1a71ec5f9f4769100f10e56c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105253779644354560","occurred_at":"2026-09-30T11:09:49Z","failed_at":"e2e","signature_id":"814a3d9ab08f714d33b125c7043504f54189a3a864c0198073bdb76d3a178e45","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c681a0\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-fl5kpw\" in resourcegroup=\"rg-np-version-upgrade-fl5kpw-96mvvhr695cd\" to finish creating: GET https://rp.j4354560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a0ac539a-44de-4d6b-be03-d9b6fc3ccdc0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a0ac539a-44de-4d6b-be03-d9b6fc3ccdc0\",\n      \"name\": \"a0ac539a-44de-4d6b-be03-d9b6fc3ccdc0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T12:36:12.039132485Z\",\n      \"endTime\": \"2026-09-30T12:55:21.071689427Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c681a0\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-fl5kpw\" in resourcegroup=\"rg-np-version-upgrade-fl5kpw-96mvvhr695cd\" to finish creating: GET https://rp.j4354560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a0ac539a-44de-4d6b-be03-d9b6fc3ccdc0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a0ac539a-44de-4d6b-be03-d9b6fc3ccdc0\",\n      \"name\": \"a0ac539a-44de-4d6b-be03-d9b6fc3ccdc0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T12:36:12.039132485Z\",\n      \"endTime\": \"2026-09-30T12:55:21.071689427Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"64c2d1d9234c34a6135ae91e01124342e7ff5dd9dc88a4e82dbf77091dc128de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7bbcc1c13014d006c1b016210955f659362471edd09845dfbd58f7a5de47ca40","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105253779644354560","occurred_at":"2026-09-30T11:09:49Z","failed_at":"e2e","signature_id":"48929c5f392dda19c39d764a9d4117d26ea3b70f835a97462a0249c92cc9fdf4","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:158]: expected all nodes to have label 'key1=value1'\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:158]: expected all nodes to have label 'key1=value1'\nExpected\n    \u003cbool\u003e: ...\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"422ddc32f327db7a61c881b4411ef7767ec79e98b46476bd6a0cffdccf708740","failure_pattern":"expected all nodes to have label 'key1=value1'","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:158]: expected all nodes to have label 'key1=value1'"}},{"row_id":"89da932e4fb012a5fe65dd67c085f4252dcdbb87a417f92e8ad888ac4fcf01a5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105253779644354560","occurred_at":"2026-09-30T11:09:49Z","failed_at":"e2e","signature_id":"95c89ae0f5cb2e646984ea99ad7d4214bf8a3269adb7da5693215c8731baef87","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000abb5a0\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-dr2mph\" in resourcegroup=\"rg-np-noedge-dr2mph-j7ng55t8mcf6\" to finish creating: GET https://rp.j4354560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/78b330e2-3532-4f23-91f1-b24676955611\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/78b330e2-3532-4f23-91f1-b24676955611\",\n      \"name\": \"78b330e2-3532-4f23-91f1-b24676955611\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T12:38:52.851263491Z\",\n      \"endTime\": \"2026-09-30T12:58:01.083392787Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000abb5a0\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-dr2mph\" in resourcegroup=\"rg-np-noedge-dr2mph-j7ng55t8mcf6\" to finish creating: GET https://rp.j4354560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/78b330e2-3532-4f23-91f1-b24676955611\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/78b330e2-3532-4f23-91f1-b24676955611\",\n      \"name\": \"78b330e2-3532-4f23-91f1-b24676955611\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T12:38:52.851263491Z\",\n      \"endTime\": \"2026-09-30T12:58:01.083392787Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"64c2d1d9234c34a6135ae91e01124342e7ff5dd9dc88a4e82dbf77091dc128de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9c90276c83c8f822e869c51bc43127253d3c5481f1f65631c683f33d3339a366","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105253779644354560","occurred_at":"2026-09-30T11:09:49Z","failed_at":"e2e","signature_id":"51a09df5028813b02368114a6b230a29919a67728e7a9fb6c2e76b59211f51e4","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b8593d4d91aeed30f93b501cb1ed65e6103f69a14e5ea5e4f7e26f7c2c0042f1","failure_pattern":"expected node pool nodes to report at least one release image ref before upgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade"}},{"row_id":"a27dbddc73d079caee9d5d24a8d5a0542aeb9a203f7e6c456afcbb92c10199e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105253779644354560","occurred_at":"2026-09-30T11:09:49Z","failed_at":"e2e","signature_id":"51a09df5028813b02368114a6b230a29919a67728e7a9fb6c2e76b59211f51e4","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:476]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:476]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b8593d4d91aeed30f93b501cb1ed65e6103f69a14e5ea5e4f7e26f7c2c0042f1","failure_pattern":"expected node pool nodes to report at least one release image ref before upgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade"}},{"row_id":"cd6a8919ca914c56894a6805380f22429dedc6fa6cf6b048342b1c85b7652ed6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105253779644354560","occurred_at":"2026-09-30T11:09:49Z","failed_at":"e2e","signature_id":"51a09df5028813b02368114a6b230a29919a67728e7a9fb6c2e76b59211f51e4","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b8593d4d91aeed30f93b501cb1ed65e6103f69a14e5ea5e4f7e26f7c2c0042f1","failure_pattern":"expected node pool nodes to report at least one release image ref before upgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade"}},{"row_id":"ea5287bbb6f52fd8ce7c07f23459d62fe096fd8dd852dfadcfb9dc8719d6e333","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105253779644354560","occurred_at":"2026-09-30T11:09:49Z","failed_at":"e2e","signature_id":"2517a1d16eaac19c98fc4a1983465611e0830eab2920dd83c5c7ac7bc707e3dd","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00082e468\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00082e468\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"ef24a3c54e88408e381821269c56e6921f93406266d98cb5a270ffad06a61906","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105253779644354560","occurred_at":"2026-09-30T11:09:49Z","failed_at":"e2e","signature_id":"31f7686d857db3ec2fac93987fae6b6a3245782f4b94f1b84b4b75f60d2d2c13","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:755]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:755]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"97763ea227d764b6cace3c59c3e2291599eb63cdd34dd1463aad690bc17db216","failure_pattern":"expected node pool nodes to report at least one release image ref before downgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:615]: expected node pool nodes to report at least one release image ref before downgrade"}}],"failure_pattern_summary":{"signature_count":7,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","422ddc32f327db7a61c881b4411ef7767ec79e98b46476bd6a0cffdccf708740","64c2d1d9234c34a6135ae91e01124342e7ff5dd9dc88a4e82dbf77091dc128de","97763ea227d764b6cace3c59c3e2291599eb63cdd34dd1463aad690bc17db216","b8593d4d91aeed30f93b501cb1ed65e6103f69a14e5ea5e4f7e26f7c2c0042f1"],"matched_occurrences":10,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105244373301071872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6900,"pr_state":"closed","pr_sha":"7551c0170c5cffea9e64f3707dcf2378a40b7d3f","final_merged_sha":"d63de0c62ddb9d7d7de369b28c8d93807d13672c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T10:32:26Z","started_at":"2026-09-30T10:32:26Z","completed_at":"2026-09-30T12:50:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T12:54:00.082357196Z","timing_metadata_checked_at":"2026-09-30T12:54:00.082357196Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T12:54:00.085735685Z","region_metadata_checked_at":"2026-09-30T12:54:00.085735685Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"73f8445ae422ae45e21aa18ed73ce4f5a4dcaf49bcc23bf952862dd41761c21f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105244373301071872","occurred_at":"2026-09-30T10:32:26Z","failed_at":"e2e","signature_id":"02b30f07b1850865809f15a839e63f1edb02365f906e6f1394faa96d6cedcc30","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-wsv85m\" for cluster \"cluster-candidate-4-23-wsv85m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f7d6c0\u003e: \n    failed to create NodePool np-wsv85m: failed waiting for nodepool=\"np-wsv85m\" for cluster \"cluster-candidate-4-23-wsv85m\" in resourcegroup=\"rg-candidate-4-23-x5s4hpmkwzhp\" to finish creating: GET https://rp.j1071872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/edf97b61-1366-48f5-936f-5378dbccb550\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/edf97b61-1366-48f5-936f-5378dbccb550\",\n      \"name\": \"edf97b61-1366-48f5-936f-5378dbccb550\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T11:35:07.81865953Z\",\n      \"endTime\": \"2026-09-30T11:54:13.175354213Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v9b2q3h5h4o9b5j-np-wsv85m-qtz79-w22nd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-x5s4hpmkwzhp-candidate-wsv85m--managed/providers/Microsoft.Compute/virtualMachines/v9b2q3h5h4o9b5j-np-wsv85m-qtz79-w22nd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v9b2q3h5h4o9b5j-np-wsv85m-qtz79-w22nd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-x5s4hpmkwzhp-candidate-wsv85m--managed/providers/Microsoft.Compute/virtualMachines/v9b2q3h5h4o9b5j-np-wsv85m-qtz79-w22nd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-wsv85m\" for cluster \"cluster-candidate-4-23-wsv85m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f7d6c0\u003e: \n    failed to create NodePool np-wsv85m: failed waiting for nodepool=\"np-wsv85m\" for cluster \"cluster-candidate-4-23-wsv85m\" in resourcegroup=\"rg-candidate-4-23-x5s4hpmkwzhp\" to finish creating: GET https://rp.j1071872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/edf97b61-1366-48f5-936f-5378dbccb550\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/edf97b61-1366-48f5-936f-5378dbccb550\",\n      \"name\": \"edf97b61-1366-48f5-936f-5378dbccb550\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T11:35:07.81865953Z\",\n      \"endTime\": \"2026-09-30T11:54:13.175354213Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v9b2q3h5h4o9b5j-np-wsv85m-qtz79-w22nd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-x5s4hpmkwzhp-candidate-wsv85m--managed/providers/Microsoft.Compute/virtualMachines/v9b2q3h5h4o9b5j-np-wsv85m-qtz79-w22nd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v9b2q3h5h4o9b5j-np-wsv85m-qtz79-w22nd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-x5s4hpmkwzhp-candidate-wsv85m--managed/providers/Microsoft.Compute/virtualMachines/v9b2q3h5h4o9b5j-np-wsv85m-qtz79-w22nd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c891abd031033d3cb3e9b62d8c25ec3e0489e46e07e6c599a1eb7e119494e08f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105244373301071872","occurred_at":"2026-09-30T10:32:26Z","failed_at":"alert","signature_id":"9e1c8df2837ce311616087aab77d355a705bb4d7065306ed80ed4c64b015fde4","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T11:34:50Z\n  Ended:       2026-09-30T11:49:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1071872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5a636m6sfjl130rh16efne40festfg-cilium-cluster\", pod=\"router-7f64d88865-r8tm4\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5a636m6sfjl130rh16efne40festfg-cilium-cluster/router-7f64d88865-r8tm4 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T11:35:51Z\n  Ended:       2026-09-30T11:49:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1071872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5a636to94rd3t38eir1d61k2r3hvq2-m4p7l3e8z9w1p5p\", pod=\"router-5cc8888886-kjp9p\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5a636to94rd3t38eir1d61k2r3hvq2-m4p7l3e8z9w1p5p/router-5cc8888886-kjp9p has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7262/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105242313000226816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7262,"pr_state":"closed","pr_sha":"4786601accbd2e57c0fa7b77dbd33b56d8d914be","final_merged_sha":"4786601accbd2e57c0fa7b77dbd33b56d8d914be","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T10:24:15Z","started_at":"2026-09-30T10:24:15Z","completed_at":"2026-09-30T12:48:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T12:54:00.226836742Z","timing_metadata_checked_at":"2026-09-30T12:54:00.226836742Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T12:54:00.229665233Z","region_metadata_checked_at":"2026-09-30T12:54:00.229665233Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105230891239346176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6884,"pr_state":"closed","pr_sha":"50a683fcdea603a7e5aec2dbc6a88e41a26babc3","final_merged_sha":"d98ef6349a3cd6d57591590dbb9cbff37ac89433","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T09:38:51Z","started_at":"2026-09-30T09:38:51Z","completed_at":"2026-09-30T12:25:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T12:32:29.943677602Z","timing_metadata_checked_at":"2026-09-30T12:32:29.943677602Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T12:32:29.951732175Z","region_metadata_checked_at":"2026-09-30T12:32:29.951732175Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"b569b8cfc428bfe4c3ff82ac7bacc196b7d088712bcd514e00748d8c2ab7daab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105230891239346176","occurred_at":"2026-09-30T09:38:51Z","failed_at":"alert","signature_id":"6ce75aaaaf15744f114f30367d6df7d93358cb13c8e5de12639712815ed34e1d","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T12:05:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j9346176-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.63:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9346176/providers/microsoft.monitor/accounts/services-j9346176\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-c94b5c67c-s5vnr\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j9346176-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T12:05:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j9346176-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.63:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9346176/providers/microsoft.monitor/accounts/services-j9346176\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-c94b5c67c-s5vnr\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j9346176-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","failure_pattern":"alert [svc] KustoLogsDataStale fired","search_query":"alert [svc] KustoLogsDataStale fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105230122981265408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6756,"pr_state":"closed","pr_sha":"c7f871c206e6784665943f544e29f3d7e87baeaa","final_merged_sha":"c7f871c206e6784665943f544e29f3d7e87baeaa","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T09:35:49Z","started_at":"2026-09-30T09:35:49Z","completed_at":"2026-09-30T12:01:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T12:11:38.009358442Z","timing_metadata_checked_at":"2026-09-30T12:11:38.009358442Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T12:11:38.01243693Z","region_metadata_checked_at":"2026-09-30T12:11:38.01243693Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105229330492690432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6662,"pr_state":"open","pr_sha":"911c458b49d2a2cd3ced7ef2fefe8affa0c9e753","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T09:32:40Z","started_at":"2026-09-30T09:32:40Z","completed_at":"2026-09-30T11:45:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T11:50:20.5164356Z","timing_metadata_checked_at":"2026-09-30T11:50:20.5164356Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T11:50:20.519141491Z","region_metadata_checked_at":"2026-09-30T11:50:20.519141491Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"566a210ed34cfc77f20ea77c2401557688fd9e23870a67a02b9acc4cec81013f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105229330492690432","occurred_at":"2026-09-30T09:32:40Z","failed_at":"e2e","signature_id":"0c99b72686df644d7a91845938f6db64d74dcf85418abd2f5d73c4d6278f406f","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bd4f00\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-l57gw6mdwbj6\" to finish creating: GET https://rp.j2690432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75db90be-5602-433c-8319-190a1fa900d9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75db90be-5602-433c-8319-190a1fa900d9\",\n      \"name\": \"75db90be-5602-433c-8319-190a1fa900d9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T10:22:37.553403502Z\",\n      \"endTime\": \"2026-09-30T10:41:41.061493076Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine d1u4m0g7s7a4h7b-nodepool-128gib-pj9gv-krcrc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-l57gw6mdwbj6--managed/providers/Microsoft.Compute/virtualMachines/d1u4m0g7s7a4h7b-nodepool-128gib-pj9gv-krcrc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine d1u4m0g7s7a4h7b-nodepool-128gib-pj9gv-krcrc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-l57gw6mdwbj6--managed/providers/Microsoft.Compute/virtualMachines/d1u4m0g7s7a4h7b-nodepool-128gib-pj9gv-krcrc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bd4f00\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-l57gw6mdwbj6\" to finish creating: GET https://rp.j2690432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75db90be-5602-433c-8319-190a1fa900d9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75db90be-5602-433c-8319-190a1fa900d9\",\n      \"name\": \"75db90be-5602-433c-8319-190a1fa900d9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T10:22:37.553403502Z\",\n      \"endTime\": \"2026-09-30T10:41:41.061493076Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine d1u4m0g7s7a4h7b-nodepool-128gib-pj9gv-krcrc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-l57gw6mdwbj6--managed/providers/Microsoft.Compute/virtualMachines/d1u4m0g7s7a4h7b-nodepool-128gib-pj9gv-krcrc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine d1u4m0g7s7a4h7b-nodepool-128gib-pj9gv-krcrc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-l57gw6mdwbj6--managed/providers/Microsoft.Compute/virtualMachines/d1u4m0g7s7a4h7b-nodepool-128gib-pj9gv-krcrc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105227914915090432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6820,"pr_state":"closed","pr_sha":"c99d95a578f89e3e4c3bfc476a031bb4b59056f4","final_merged_sha":"273677ce69aa7a8dc7e1da98b8942d3ea48b24d1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T09:27:02Z","started_at":"2026-09-30T09:27:02Z","completed_at":"2026-09-30T10:38:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T10:56:56.826576217Z","timing_metadata_checked_at":"2026-09-30T10:56:56.826576217Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T10:56:56.829552107Z","region_metadata_checked_at":"2026-09-30T10:56:56.829552107Z"},"failed_at":["alert","provision"],"failed_test_count":4,"occurrences":[{"row_id":"08a39272e9d55a927edf37292e4efdb7743e386b13006c794b70d3c46611f830","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105227914915090432","occurred_at":"2026-09-30T09:27:02Z","failed_at":"provision","signature_id":"d20ad03882fd0f5debbe72a144cd887757a3333ce3a6250ab7b88059925a6396","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","test_suite":"templatize-pipeline","failure_text":"time=2026-09-30T10:02:16.720Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-30T10:02:17.023Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j5090432-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:eb3e2e068172121a9ca133c5d17784df770dacc104f20c7d4a7df5f1c70e8a08 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:450m memory:472Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j5090432-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5c29681bb4b38d08ca21c31c4553bab72e8e9c260dc0ec875e1d014268a9e580 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:e5a2e4f0-d338-424f-81b6-3ea36e90ffa8 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#e5a2e4f0-d338-424f-81b6-3ea36e90ffa8 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-30T10:02:17.023Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T10:02:17.180Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-30T10:02:17.354Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-30T10:02:17.354Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T10:02:17.402Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T10:12:19.938Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T10:12:19.938Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T10:13:19.944Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-30T10:13:20.698Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j5090432-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:eb3e2e068172121a9ca133c5d17784df770dacc104f20c7d4a7df5f1c70e8a08 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:450m memory:472Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j5090432-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5c29681bb4b38d08ca21c31c4553bab72e8e9c260dc0ec875e1d014268a9e580 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:e5a2e4f0-d338-424f-81b6-3ea36e90ffa8 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#e5a2e4f0-d338-424f-81b6-3ea36e90ffa8 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-30T10:13:20.699Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T10:13:21.000Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-30T10:13:21.389Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T10:13:21.389Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T10:13:21.559Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-30T10:13:22.194Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T10:13:22.194Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T10:13:22.766Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T10:13:23.138Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T10:13:23.614Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T10:13:24.021Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T10:13:24.220Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T10:13:24.545Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T10:13:24.923Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-30T10:13:24.923Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T10:13:25.004Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-30T10:23:26.540Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T10:23:26.540Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T10:24:26.541Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-30T10:24:26.978Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j5090432-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:eb3e2e068172121a9ca133c5d17784df770dacc104f20c7d4a7df5f1c70e8a08 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:450m memory:472Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j5090432-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5c29681bb4b38d08ca21c31c4553bab72e8e9c260dc0ec875e1d014268a9e580 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:e5a2e4f0-d338-424f-81b6-3ea36e90ffa8 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#e5a2e4f0-d338-424f-81b6-3ea36e90ffa8 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-30T10:24:26.978Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T10:24:27.205Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-30T10:24:27.597Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T10:24:27.597Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T10:24:27.807Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-30T10:24:28.587Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T10:24:28.587Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T10:24:28.952Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T10:24:29.210Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T10:24:29.577Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T10:24:29.759Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T10:24:30.112Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T10:24:30.448Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T10:24:30.762Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-30T10:24:30.762Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T10:24:30.844Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-30T10:32:16.654Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"context deadline exceeded\"\ntime=2026-09-30T10:32:16.654Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","failure_pattern":"failed to roll out Helm release: context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded"}},{"row_id":"53f100d34948cd9dc6986263ea2fd212ca3e655ea8fc80f127573cd11309a2b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105227914915090432","occurred_at":"2026-09-30T09:27:02Z","failed_at":"alert","signature_id":"69c10044ecef2b03f895153e90c169d4ea0567b1642ce13c7454d66e14ce62ee","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:19:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j5090432-svc\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"warning\"\n  Description: The Arobit forwarder metrics endpoint on cluster ci01-j5090432-svc has been unreachable for 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired"}},{"row_id":"b19de7a1cbb4b48460128795333acfd308de5a4e6ae9b5d5af73da0c894abadf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105227914915090432","occurred_at":"2026-09-30T09:27:02Z","failed_at":"alert","signature_id":"4ef2e890beff7c4f86db4449f89189712a00b70740bd3d6bf71d4bd36e6eada0","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:08:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5090432-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-c997l\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-c997l has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:08:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5090432-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-qptj8\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-qptj8 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:08:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5090432-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-69sbs\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-69sbs has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"b4163a83cb2d9a4d62d63b769042e0c38a2c05be0bb8b00e8f0941330352b7f5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105227914915090432","occurred_at":"2026-09-30T09:27:02Z","failed_at":"alert","signature_id":"070a999d413963099d095402bcad5d0495e08e705056ebb3a3163bfbe8e380fa","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:18:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j5090432-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.69:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5090432/providers/microsoft.monitor/accounts/services-j5090432\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-gmtlp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:18:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j5090432-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.69:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5090432/providers/microsoft.monitor/accounts/services-j5090432\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-gmtlp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6919/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105227543996010496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6919,"pr_state":"closed","pr_sha":"f5f7ba3708403bd680936e8b420e0d61595cc458","final_merged_sha":"f5f7ba3708403bd680936e8b420e0d61595cc458","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T09:25:34Z","started_at":"2026-09-30T09:25:34Z","completed_at":"2026-09-30T11:58:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T12:00:51.780522186Z","timing_metadata_checked_at":"2026-09-30T12:00:51.780522186Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T12:00:51.782843178Z","region_metadata_checked_at":"2026-09-30T12:00:51.782843178Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105227448479125504","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6993,"pr_state":"closed","pr_sha":"54fa83b6be2bda741a3e467491a8b8caf8a2dbe3","final_merged_sha":"1a558ec553718fc2df21030c08516ca31075af74","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T09:25:11Z","started_at":"2026-09-30T09:25:11Z","completed_at":"2026-09-30T11:56:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T12:00:51.925279024Z","timing_metadata_checked_at":"2026-09-30T12:00:51.925279024Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T12:00:51.927668216Z","region_metadata_checked_at":"2026-09-30T12:00:51.927668216Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7220/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105227431290867712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7220,"pr_state":"open","pr_sha":"540856c435b752648f8f78fe4a157f67de6eed0a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T09:25:07Z","started_at":"2026-09-30T09:25:07Z","completed_at":"2026-09-30T11:47:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T11:50:21.052669628Z","timing_metadata_checked_at":"2026-09-30T11:50:21.052669628Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T11:50:21.059459806Z","region_metadata_checked_at":"2026-09-30T11:50:21.059459806Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"4c32f3295b9628c046c828dd1281cb19208d6b85b2f7a37ba55f404911daa4a0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7220/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105227431290867712","occurred_at":"2026-09-30T09:25:07Z","failed_at":"e2e","signature_id":"f705bab8d4be596b76259e82bbce6d86611afa3edaf3393759077548f5abd2e5","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 611.396s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:157 with:\nRunVMCommand failed for ingress connectivity test\nUnexpected error:\n    \u003c*errors.errorString | 0xc00158e610\u003e: \n    run command \"e2e-runcommand-1790764865640292876\" failed with exit code 28 on VM \"private-ingress-test-vm\" (output: \"000\")\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 611.396s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:157 with:\nRunVMCommand failed for ingress connectivity test\nUnexpected error:\n    \u003c*errors.errorString | 0xc00158e610\u003e: \n    run command \"e2e-runcommand-1790764865640292876\" failed with exit code 28 on VM \"private-ingress-test-vm\" (output: \"000\")\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bb120c9514f41ade3798c0918bf6c7e33712fa0ac08db0b23b0950e854a6cf03","failure_pattern":"run command \"e2e-runcommand-1790764865640292876\" failed with exit code 28 on VM \"\u003cvm\u003e\" (output: \"000\")","search_query":"run command \"e2e-runcommand-1790764865640292876\" failed with exit code 28 on VM \"private-ingress-test-vm\" (output: \"000\")"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["bb120c9514f41ade3798c0918bf6c7e33712fa0ac08db0b23b0950e854a6cf03"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7071/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105227392430641152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7071,"pr_state":"open","pr_sha":"6e66d4c793650a413acda3f74dda6a03700e788a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T09:24:58Z","started_at":"2026-09-30T09:24:58Z","completed_at":"2026-09-30T10:22:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T10:35:01.316906616Z","timing_metadata_checked_at":"2026-09-30T10:35:01.316906616Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T10:35:01.319858106Z","region_metadata_checked_at":"2026-09-30T10:35:01.319858106Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"eda3d156511d7212744adbc99bfb6339be0dc2d6e6823a0fb19889471bf6bad6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7071/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105227392430641152","occurred_at":"2026-09-30T09:24:58Z","failed_at":"provision","signature_id":"19498df56e93c37fe7f6fb08ad99bc2b1b1c8d05100b0a59886217e4fe1408e0","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/storageclass (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-30T10:09:47.479Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 description=\"Step storageclass\\n Kind: Helm\\n\"\ntime=2026-09-30T10:09:48.123Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 values=\"map[skuName:PremiumV2_LRS unannotateJobImage:mcr.microsoft.com/aks/command/runtime@sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4]\"\ntime=2026-09-30T10:09:48.123Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-09-30T10:09:48.426Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 namespace=default-sc\ntime=2026-09-30T10:09:48.816Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 dryRun=false\ntime=2026-09-30T10:09:48.816Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-09-30T10:09:48.942Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-09-30T10:14:53.371Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 err=\"failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded\"\ntime=2026-09-30T10:14:53.371Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"abcd5f9b62e8cf7d13105a0c9ded33446cbd1e1f876e962101b7bbf9a2c604c6","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: InProgress, message: Job in progress context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["abcd5f9b62e8cf7d13105a0c9ded33446cbd1e1f876e962101b7bbf9a2c604c6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105226917077585920","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7112,"pr_state":"closed","pr_sha":"395e45d0bbde804a9c22a549e978e9371ff65d91","final_merged_sha":"c11d1aab795ebbd762ddef1d8188a105d875d002","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T09:23:03Z","started_at":"2026-09-30T09:23:03Z","completed_at":"2026-09-30T11:45:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T11:50:20.756488307Z","timing_metadata_checked_at":"2026-09-30T11:50:20.756488307Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T11:50:20.759215798Z","region_metadata_checked_at":"2026-09-30T11:50:20.759215798Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"4de2c49c7698f94e5615fcb09cdebf543d09150287ec5f69dcad0ae2dccec1ad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105226917077585920","occurred_at":"2026-09-30T09:23:03Z","failed_at":"e2e","signature_id":"4cbd0b39303c5ba882664f24927fa89f1b1e0d8bda43db16774956c03f9a6c68","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-7jbkqj\" with version 4.21.34 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011488e0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-7jbkqj: failed waiting for cluster=\"cluster-zstream-4-21-7jbkqj\" in resourcegroup=\"rg-zstream-upgrade-4-21-7qqcshqrm588\" to finish creating: GET https://rp.j7585920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/30ab22cf-79b3-49a2-b5a2-b420840f6f78\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/30ab22cf-79b3-49a2-b5a2-b420840f6f78' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/30ab22cf-79b3-49a2-b5a2-b420840f6f78\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-7jbkqj\" with version 4.21.34 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011488e0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-7jbkqj: failed waiting for cluster=\"cluster-zstream-4-21-7jbkqj\" in resourcegroup=\"rg-zstream-upgrade-4-21-7qqcshqrm588\" to finish creating: GET https://rp.j7585920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/30ab22cf-79b3-49a2-b5a2-b420840f6f78\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/30ab22cf-79b3-49a2-b5a2-b420840f6f78' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/30ab22cf-79b3-49a2-b5a2-b420840f6f78\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7176/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105226057555644416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7176,"pr_state":"closed","pr_sha":"a90ad015f6e70d63790d33a15a5294c05dbf614b","final_merged_sha":"a90ad015f6e70d63790d33a15a5294c05dbf614b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T09:19:39Z","started_at":"2026-09-30T09:19:39Z","completed_at":"2026-09-30T11:45:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T11:50:20.918556771Z","timing_metadata_checked_at":"2026-09-30T11:50:20.918556771Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T11:50:20.92190686Z","region_metadata_checked_at":"2026-09-30T11:50:20.92190686Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105225624229515264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7068,"pr_state":"closed","pr_sha":"5d483559deea65ed903f146fc4f20cbbc522559d","final_merged_sha":"269e5d56bcfac6dc9f8d3ca87b7fb1ea84bf53b2","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T09:17:55Z","started_at":"2026-09-30T09:17:55Z","completed_at":"2026-09-30T11:53:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T12:00:52.067749269Z","timing_metadata_checked_at":"2026-09-30T12:00:52.067749269Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T12:00:52.070315161Z","region_metadata_checked_at":"2026-09-30T12:00:52.070315161Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7229/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105225488111767552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7229,"pr_state":"closed","pr_sha":"e251feee783228a94dd7ed8590b63c06c237cbb6","final_merged_sha":"e251feee783228a94dd7ed8590b63c06c237cbb6","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T09:17:24Z","started_at":"2026-09-30T09:17:24Z","completed_at":"2026-09-30T11:58:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T12:00:52.210386913Z","timing_metadata_checked_at":"2026-09-30T12:00:52.210386913Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T12:00:52.215980295Z","region_metadata_checked_at":"2026-09-30T12:00:52.215980295Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105225367915597824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6915,"pr_state":"open","pr_sha":"0bdd36fc9bed8d937676c8592b848cd9cbcde70d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T09:16:55Z","started_at":"2026-09-30T09:16:55Z","completed_at":"2026-09-30T11:51:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T12:00:51.61643531Z","timing_metadata_checked_at":"2026-09-30T12:00:51.61643531Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T12:00:51.619536Z","region_metadata_checked_at":"2026-09-30T12:00:51.619536Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"81d160828454cbc25ad9e4d32319090ce9770a5d36b8678570dbd5dee05cee99","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105225367915597824","occurred_at":"2026-09-30T09:16:55Z","failed_at":"e2e","signature_id":"0004ec9246093dd32ccef6900517fe1c26f01933d048bec9e64d50cce9f0f872","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-xkg7gf\" for cluster \"cluster-candidate-4-20-xkg7gf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001435b60\u003e: \n    failed to create NodePool np-xkg7gf: failed waiting for nodepool=\"np-xkg7gf\" for cluster \"cluster-candidate-4-20-xkg7gf\" in resourcegroup=\"rg-candidate-4-20-j47p9rbxx46g\" to finish creating: GET https://rp.j5597824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26468792-a524-4bf9-970b-d4edc2f9bfe8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26468792-a524-4bf9-970b-d4edc2f9bfe8\",\n      \"name\": \"26468792-a524-4bf9-970b-d4edc2f9bfe8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T10:25:29.553885983Z\",\n      \"endTime\": \"2026-09-30T10:44:30.082076643Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q2o4d5p5y8u3o9e-np-xkg7gf-jxzwr-xwjrg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-j47p9rbxx46g-candidate-xkg7gf--managed/providers/Microsoft.Compute/virtualMachines/q2o4d5p5y8u3o9e-np-xkg7gf-jxzwr-xwjrg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q2o4d5p5y8u3o9e-np-xkg7gf-jxzwr-xwjrg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-j47p9rbxx46g-candidate-xkg7gf--managed/providers/Microsoft.Compute/virtualMachines/q2o4d5p5y8u3o9e-np-xkg7gf-jxzwr-xwjrg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-xkg7gf\" for cluster \"cluster-candidate-4-20-xkg7gf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001435b60\u003e: \n    failed to create NodePool np-xkg7gf: failed waiting for nodepool=\"np-xkg7gf\" for cluster \"cluster-candidate-4-20-xkg7gf\" in resourcegroup=\"rg-candidate-4-20-j47p9rbxx46g\" to finish creating: GET https://rp.j5597824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26468792-a524-4bf9-970b-d4edc2f9bfe8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26468792-a524-4bf9-970b-d4edc2f9bfe8\",\n      \"name\": \"26468792-a524-4bf9-970b-d4edc2f9bfe8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T10:25:29.553885983Z\",\n      \"endTime\": \"2026-09-30T10:44:30.082076643Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q2o4d5p5y8u3o9e-np-xkg7gf-jxzwr-xwjrg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-j47p9rbxx46g-candidate-xkg7gf--managed/providers/Microsoft.Compute/virtualMachines/q2o4d5p5y8u3o9e-np-xkg7gf-jxzwr-xwjrg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q2o4d5p5y8u3o9e-np-xkg7gf-jxzwr-xwjrg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-j47p9rbxx46g-candidate-xkg7gf--managed/providers/Microsoft.Compute/virtualMachines/q2o4d5p5y8u3o9e-np-xkg7gf-jxzwr-xwjrg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6944/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105224754142121984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6944,"pr_state":"open","pr_sha":"d00b03862945426285f54b76cb1f24ccfc8fdbcb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T09:14:29Z","started_at":"2026-09-30T09:14:29Z","completed_at":"2026-09-30T10:12:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T10:24:49.799005238Z","timing_metadata_checked_at":"2026-09-30T10:24:49.799005238Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T10:24:49.80462712Z","region_metadata_checked_at":"2026-09-30T10:24:49.80462712Z"},"failed_at":["alert","provision"],"failed_test_count":2,"occurrences":[{"row_id":"473175968bc09b55161db8fa2d0daffee39ba4bbbe5274eae9a86569cafb3a19","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6944/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105224754142121984","occurred_at":"2026-09-30T09:14:29Z","failed_at":"provision","signature_id":"72c70d03904798e23a81b751f90dae41f83d98201724acd02612ebc1b536daae","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Backend/service/deploy","test_suite":"templatize-pipeline","failure_text":"time=2026-09-30T10:01:25.496Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-09-30T10:01:25.890Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[armPermissionsManagerIdentity:map[certName:armHelperPoolCert-34 clientId:72671881-beb9-4028-bc7c-47f511281566 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpci01oidcj2121984.z9.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpci01oidcj2121984.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] backupCadenceProfile:testing backupScheduleState:Disabled clusterScopedIdentitiesConfig:map[roleSetName:dev] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:072ccd06-457a-49cc-a612-ad4dbbc199a3 databaseName:arohcpci01-rp-j2121984 databaseUrl:https://arohcpci01-rp-j2121984.documents.azure.com:443/ kubeApplierCosmosContainerName:Manifests-MC-1 location:canadacentral] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-gq2sycdx/pipeline@sha256:ec35d9956c06798bf0f6c7e4116d8b1c120ff9a5a7cbe4e991d36cae1bea464e replicas:2 resources:map[limits:map[memory:unlimited] requests:map[cpu:700m memory:1Gi]] zoneCount:3] exitOnPanic:true fpa:map[certName:firstPartyCert2 clientId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] logVerbosity:4 maestro:map[sourceEnvironmentIdentifier:arohcpci01] miMock:map[certName:msiMockPoolCert-18 clientId:8c088886-a118-4973-8168-0d7ae1493aa7 clientIdSecretName: principalId:ccbf438f-08ec-4e2b-ae4e-219ce7dc3762 principalIdSecretName: tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-gq2sycdx/pipeline:pull workloadIdentityClientId:7cbf62ec-0a3a-4039-a0c0-7c88aa6f75bf workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:072ccd06-457a-49cc-a612-ad4dbbc199a3 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv]]\"\ntime=2026-09-30T10:01:25.890Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-09-30T10:01:26.074Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-09-30T10:01:26.234Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-09-30T10:01:26.234Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-09-30T10:01:26.298Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-09-30T10:06:31.687Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 0/2\\ncontext deadline exceeded\"\ntime=2026-09-30T10:06:31.687Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 0/2\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e8bcec5029342ecf0af7106c015ff83add435f419b54399997693a996c71892f","failure_pattern":"failed to roll out Helm release: resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 0/2 context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 0/2\\ncontext deadline exceeded"}},{"row_id":"75fb01d4f64918af81a1a83d534a6c01bfe962c6974dec4710ba836a01c9ad01","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6944/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105224754142121984","occurred_at":"2026-09-30T09:14:29Z","failed_at":"alert","signature_id":"960817798cecef5f242862956f5ea60bc993965ed8ff7b4209f48c2ae021c775","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 95 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-rsvtx\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-rsvtx has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-796x2\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-796x2 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-pcjcl\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-pcjcl has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-z998l\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-z998l has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-m8698\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-m8698 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-82cr4\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-82cr4 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-pzjb4\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-pzjb4 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-7x96q\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-7x96q has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-x2wph\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-x2wph has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-n9dj7\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-n9dj7 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-nwhcl\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-nwhcl has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-vgjxq\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-vgjxq has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-5nrdf\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-5nrdf has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-m52wd\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-m52wd has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-5g7dm\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-5g7dm has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-sqlsb\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-sqlsb has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-vfwjz\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-vfwjz has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-czw9d\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-czw9d has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-ft9sh\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-ft9sh has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-2lclr\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-2lclr has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-tgq28\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-tgq28 has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-qvg8l\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-qvg8l has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-js2ms\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-js2ms has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-s2mtg\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-s2mtg has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-g76kv\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-g76kv has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-6rcxw\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-6rcxw has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-sw7b9\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-sw7b9 has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-c4n6b\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-c4n6b has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-lq6ck\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-lq6ck has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-dvlrh\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-dvlrh has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-cj9md\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-cj9md has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-bjcl4\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-bjcl4 has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-48gt2\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-48gt2 has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-qchw5\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-qchw5 has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-bz68s\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-bz68s has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-tv8hg\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-tv8hg has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-8n25z\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-8n25z has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-n4r6z\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-n4r6z has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-gpj9m\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-gpj9m has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-f4z6c\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-f4z6c has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-cd9g2\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-cd9g2 has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-vqj8z\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-vqj8z has been in a non-ready state for longer than 5 minutes.\nFiring 43:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-zm597\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-zm597 has been in a non-ready state for longer than 5 minutes.\nFiring 44:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-q6m5f\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-q6m5f has been in a non-ready state for longer than 5 minutes.\nFiring 45:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-bxhlp\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-bxhlp has been in a non-ready state for longer than 5 minutes.\nFiring 46:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-8c6pj\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-8c6pj has been in a non-ready state for longer than 5 minutes.\nFiring 47:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-gjs74\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-gjs74 has been in a non-ready state for longer than 5 minutes.\nFiring 48:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-lrcrl\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-lrcrl has been in a non-ready state for longer than 5 minutes.\nFiring 49:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-kwfr5\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-kwfr5 has been in a non-ready state for longer than 5 minutes.\nFiring 50:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-jnhrj\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-jnhrj has been in a non-ready state for longer than 5 minutes.\nFiring 51:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-j2x54\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-j2x54 has been in a non-ready state for longer than 5 minutes.\nFiring 52:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-m4w5z\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-m4w5z has been in a non-ready state for longer than 5 minutes.\nFiring 53:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-c5jzp\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-c5jzp has been in a non-ready state for longer than 5 minutes.\nFiring 54:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-xtl8c\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-xtl8c has been in a non-ready state for longer than 5 minutes.\nFiring 55:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-pk965\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-pk965 has been in a non-ready state for longer than 5 minutes.\nFiring 56:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-xc2p8\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-xc2p8 has been in a non-ready state for longer than 5 minutes.\nFiring 57:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-jkhgh\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-jkhgh has been in a non-ready state for longer than 5 minutes.\nFiring 58:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-vl6w8\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-vl6w8 has been in a non-ready state for longer than 5 minutes.\nFiring 59:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-9xhbm\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-9xhbm has been in a non-ready state for longer than 5 minutes.\nFiring 60:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-pldcj\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-pldcj has been in a non-ready state for longer than 5 minutes.\nFiring 61:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-wzvnf\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-wzvnf has been in a non-ready state for longer than 5 minutes.\nFiring 62:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-c8pn2\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-c8pn2 has been in a non-ready state for longer than 5 minutes.\nFiring 63:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-7dgv9\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-7dgv9 has been in a non-ready state for longer than 5 minutes.\nFiring 64:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-cw6k5\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-cw6k5 has been in a non-ready state for longer than 5 minutes.\nFiring 65:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-cp898\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-cp898 has been in a non-ready state for longer than 5 minutes.\nFiring 66:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-vfv5j\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-vfv5j has been in a non-ready state for longer than 5 minutes.\nFiring 67:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-mcbpb\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-mcbpb has been in a non-ready state for longer than 5 minutes.\nFiring 68:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-fq9qk\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-fq9qk has been in a non-ready state for longer than 5 minutes.\nFiring 69:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-v2www\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-v2www has been in a non-ready state for longer than 5 minutes.\nFiring 70:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-8j9r8\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-8j9r8 has been in a non-ready state for longer than 5 minutes.\nFiring 71:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-dtv87\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-dtv87 has been in a non-ready state for longer than 5 minutes.\nFiring 72:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-pk6pc\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-pk6pc has been in a non-ready state for longer than 5 minutes.\nFiring 73:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-jpfz4\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-jpfz4 has been in a non-ready state for longer than 5 minutes.\nFiring 74:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-v6ml2\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-v6ml2 has been in a non-ready state for longer than 5 minutes.\nFiring 75:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-msxpq\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-msxpq has been in a non-ready state for longer than 5 minutes.\nFiring 76:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-hbpsf\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-hbpsf has been in a non-ready state for longer than 5 minutes.\nFiring 77:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-5njm9\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-5njm9 has been in a non-ready state for longer than 5 minutes.\nFiring 78:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-2nc5h\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-2nc5h has been in a non-ready state for longer than 5 minutes.\nFiring 79:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-7ftmt\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-7ftmt has been in a non-ready state for longer than 5 minutes.\nFiring 80:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-ccxrv\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-ccxrv has been in a non-ready state for longer than 5 minutes.\nFiring 81:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-ngg9z\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-ngg9z has been in a non-ready state for longer than 5 minutes.\nFiring 82:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-8w6bk\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-8w6bk has been in a non-ready state for longer than 5 minutes.\nFiring 83:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-jgzjj\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-jgzjj has been in a non-ready state for longer than 5 minutes.\nFiring 84:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-4dchf\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-4dchf has been in a non-ready state for longer than 5 minutes.\nFiring 85:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-dkq6z\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-dkq6z has been in a non-ready state for longer than 5 minutes.\nFiring 86:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-vvrhd\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-vvrhd has been in a non-ready state for longer than 5 minutes.\nFiring 87:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-fllz9\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-fllz9 has been in a non-ready state for longer than 5 minutes.\nFiring 88:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-jckvz\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-jckvz has been in a non-ready state for longer than 5 minutes.\nFiring 89:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-2fn44\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-2fn44 has been in a non-ready state for longer than 5 minutes.\nFiring 90:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-k5psj\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-k5psj has been in a non-ready state for longer than 5 minutes.\nFiring 91:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-4r64q\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-4r64q has been in a non-ready state for longer than 5 minutes.\nFiring 92:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-98h9z\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-98h9z has been in a non-ready state for longer than 5 minutes.\nFiring 93:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-9b6sv\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-9b6sv has been in a non-ready state for longer than 5 minutes.\nFiring 94:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-sfw6f\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-sfw6f has been in a non-ready state for longer than 5 minutes.\nFiring 95:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-6285h\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-6285h has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","e8bcec5029342ecf0af7106c015ff83add435f419b54399997693a996c71892f"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7221/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105221039314178048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7221,"pr_state":"closed","pr_sha":"d873d5fd809e699ac864420434c1824fcabd44c1","final_merged_sha":"d873d5fd809e699ac864420434c1824fcabd44c1","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T08:59:43Z","started_at":"2026-09-30T08:59:43Z","completed_at":"2026-09-30T11:24:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T11:39:37.613479559Z","timing_metadata_checked_at":"2026-09-30T11:39:37.613479559Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T11:39:37.618252442Z","region_metadata_checked_at":"2026-09-30T11:39:37.618252442Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7104/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105219090921558016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7104,"pr_state":"closed","pr_sha":"a2e35daa20b37841bf1d2f73e88d95554cdc5664","final_merged_sha":"a2e35daa20b37841bf1d2f73e88d95554cdc5664","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T08:51:58Z","started_at":"2026-09-30T08:51:58Z","completed_at":"2026-09-30T11:14:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T11:29:06.920150004Z","timing_metadata_checked_at":"2026-09-30T11:29:06.920150004Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T11:29:06.924542991Z","region_metadata_checked_at":"2026-09-30T11:29:06.924542991Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7262/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105203604242567168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7262,"pr_state":"closed","pr_sha":"4786601accbd2e57c0fa7b77dbd33b56d8d914be","final_merged_sha":"4786601accbd2e57c0fa7b77dbd33b56d8d914be","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T07:50:26Z","started_at":"2026-09-30T07:50:26Z","completed_at":"2026-09-30T10:13:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T10:24:49.951983251Z","timing_metadata_checked_at":"2026-09-30T10:24:49.951983251Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T10:24:49.95531914Z","region_metadata_checked_at":"2026-09-30T10:24:49.95531914Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"71a0a4c4672902b2859d1d4f3ac27c03ce51222006fc99ec96d050216667aa0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7262/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105203604242567168","occurred_at":"2026-09-30T07:50:26Z","failed_at":"e2e","signature_id":"7c21d39ce60d2fc86f8e44cd27976742c9868e16e557b49cd66e0e9f7da03ea6","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:205]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002511248\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-xk42m82bkxtn\" to finish creating: GET https://rp.j2567168.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/034a0762-ab5c-48da-87dd-8d208861a970\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/034a0762-ab5c-48da-87dd-8d208861a970\",\n      \"name\": \"034a0762-ab5c-48da-87dd-8d208861a970\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:55:45.833258741Z\",\n      \"endTime\": \"2026-09-30T09:14:52.939311971Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-nssc6-7nz65: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xk42m82bkxtn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-nssc6-7nz65 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-nssc6-7nz65: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xk42m82bkxtn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-nssc6-7nz65 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    encountered errors downloading console logs:\n      - VM \"cilium-cluster-cilium-np-nssc6-7nz65\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-nssc6-7nz65\" after 3 attempts, last error: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xk42m82bkxtn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-nssc6-7nz65/retrieveBootDiagnosticsData\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: OperationNotAllowed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"OperationNotAllowed\",\n        \"message\": \"Operation 'retrieveBootDiagnosticsData' is not allowed on VM 'cilium-cluster-cilium-np-nssc6-7nz65' since another operation is in progress.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:205]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002511248\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-xk42m82bkxtn\" to finish creating: GET https://rp.j2567168.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/034a0762-ab5c-48da-87dd-8d208861a970\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/034a0762-ab5c-48da-87dd-8d208861a970\",\n      \"name\": \"034a0762-ab5c-48da-87dd-8d208861a970\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:55:45.833258741Z\",\n      \"endTime\": \"2026-09-30T09:14:52.939311971Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-nssc6-7nz65: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xk42m82bkxtn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-nssc6-7nz65 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-nssc6-7nz65: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xk42m82bkxtn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-nssc6-7nz65 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    encountered errors downloading console logs:\n      - VM \"cilium-cluster-cilium-np-nssc6-7nz65\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-nssc6-7nz65\" after 3 attempts, last error: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xk42m82bkxtn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-nssc6-7nz65/retrieveBootDiagnosticsData\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: OperationNotAllowed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"OperationNotAllowed\",\n        \"message\": \"Operation 'retrieveBootDiagnosticsData' is not allowed on VM 'cilium-cluster-cilium-np-nssc6-7nz65' since another operation is in progress.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105193266038181888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6884,"pr_state":"closed","pr_sha":"50a683fcdea603a7e5aec2dbc6a88e41a26babc3","final_merged_sha":"d98ef6349a3cd6d57591590dbb9cbff37ac89433","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T07:09:21Z","started_at":"2026-09-30T07:09:21Z","completed_at":"2026-09-30T09:29:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T09:42:58.821221896Z","timing_metadata_checked_at":"2026-09-30T09:42:58.821221896Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T09:42:58.824746685Z","region_metadata_checked_at":"2026-09-30T09:42:58.824746685Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"52b438b71436b28a7d3ad3bdf1d1b7bc36afe9b237858b94d05937154b109a4a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105193266038181888","occurred_at":"2026-09-30T07:09:21Z","failed_at":"e2e","signature_id":"64488f0586b5477bcb601e48c11446bba99626aed3e38dfbb290b68d5bcdd29e","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aae2c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-nkgjwtvv229n\" to finish creating: GET https://rp.j8181888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fc933c7-0d41-4651-a4a8-b2c8e1264ee6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fc933c7-0d41-4651-a4a8-b2c8e1264ee6\",\n      \"name\": \"4fc933c7-0d41-4651-a4a8-b2c8e1264ee6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:33:01.043737315Z\",\n      \"endTime\": \"2026-09-30T08:52:07.873742657Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j6f0a3z3m5a4v2o-np-1-cdxdq-nmfh9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-nkgjwtvv229n--managed/providers/Microsoft.Compute/virtualMachines/j6f0a3z3m5a4v2o-np-1-cdxdq-nmfh9 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j6f0a3z3m5a4v2o-np-1-cdxdq-nmfh9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-nkgjwtvv229n--managed/providers/Microsoft.Compute/virtualMachines/j6f0a3z3m5a4v2o-np-1-cdxdq-nmfh9 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aae2c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-nkgjwtvv229n\" to finish creating: GET https://rp.j8181888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fc933c7-0d41-4651-a4a8-b2c8e1264ee6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fc933c7-0d41-4651-a4a8-b2c8e1264ee6\",\n      \"name\": \"4fc933c7-0d41-4651-a4a8-b2c8e1264ee6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:33:01.043737315Z\",\n      \"endTime\": \"2026-09-30T08:52:07.873742657Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j6f0a3z3m5a4v2o-np-1-cdxdq-nmfh9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-nkgjwtvv229n--managed/providers/Microsoft.Compute/virtualMachines/j6f0a3z3m5a4v2o-np-1-cdxdq-nmfh9 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j6f0a3z3m5a4v2o-np-1-cdxdq-nmfh9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-nkgjwtvv229n--managed/providers/Microsoft.Compute/virtualMachines/j6f0a3z3m5a4v2o-np-1-cdxdq-nmfh9 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105192683818455040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6922,"pr_state":"closed","pr_sha":"e772682c6ada7bf3fd5674946d72fe25f79d834b","final_merged_sha":"e772682c6ada7bf3fd5674946d72fe25f79d834b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T07:06:56Z","started_at":"2026-09-30T07:06:56Z","completed_at":"2026-09-30T09:27:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T09:42:58.841921029Z","timing_metadata_checked_at":"2026-09-30T09:42:58.841921029Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T09:42:58.844741519Z","region_metadata_checked_at":"2026-09-30T09:42:58.844741519Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7182/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105191830118207488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7182,"pr_state":"closed","pr_sha":"b56a252408226d8ed3ec180421d2845783ace28c","final_merged_sha":"b56a252408226d8ed3ec180421d2845783ace28c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T07:03:39Z","started_at":"2026-09-30T07:03:39Z","completed_at":"2026-09-30T09:26:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T09:42:58.861493365Z","timing_metadata_checked_at":"2026-09-30T09:42:58.861493365Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T09:42:58.864362055Z","region_metadata_checked_at":"2026-09-30T09:42:58.864362055Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7260/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105189267666898944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7260,"pr_state":"closed","pr_sha":"d2063f84c5105758d4ff2370d77bd0c238c14b4d","final_merged_sha":"e2029a06a51da6cdf2ed5b5ef1c620dcc9cda152","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T06:53:28Z","started_at":"2026-09-30T06:53:28Z","completed_at":"2026-09-30T09:17:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T09:32:36.719167257Z","timing_metadata_checked_at":"2026-09-30T09:32:36.719167257Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T09:32:36.722742946Z","region_metadata_checked_at":"2026-09-30T09:32:36.722742946Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"a98b5e849cc54afb43c09f0ac45ea0cf93e44f1c4ff1cfa3a1d7b10ff53b2af5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7260/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105189267666898944","occurred_at":"2026-09-30T06:53:28Z","failed_at":"e2e","signature_id":"87fbac54cf2eeb9a961209caab24269852fdc963c5e9568aa686fbe5fb493555","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000c97540\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-4vhtkz65lcdf\" to finish creating: GET https://rp.j6898944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f923a9cc-3bf2-46c5-a14f-9498c8c16cec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f923a9cc-3bf2-46c5-a14f-9498c8c16cec\",\n      \"name\": \"f923a9cc-3bf2-46c5-a14f-9498c8c16cec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:58:35.462676175Z\",\n      \"endTime\": \"2026-09-30T08:17:44.959895115Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v6y0n4l6q1w8i3p-np-delete-main-p2bwr-dv274: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-4vhtkz65lcdf--managed/providers/Microsoft.Compute/virtualMachines/v6y0n4l6q1w8i3p-np-delete-main-p2bwr-dv274 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v6y0n4l6q1w8i3p-np-delete-main-p2bwr-dv274: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-4vhtkz65lcdf--managed/providers/Microsoft.Compute/virtualMachines/v6y0n4l6q1w8i3p-np-delete-main-p2bwr-dv274 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000c97540\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-4vhtkz65lcdf\" to finish creating: GET https://rp.j6898944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f923a9cc-3bf2-46c5-a14f-9498c8c16cec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f923a9cc-3bf2-46c5-a14f-9498c8c16cec\",\n      \"name\": \"f923a9cc-3bf2-46c5-a14f-9498c8c16cec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:58:35.462676175Z\",\n      \"endTime\": \"2026-09-30T08:17:44.959895115Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v6y0n4l6q1w8i3p-np-delete-main-p2bwr-dv274: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-4vhtkz65lcdf--managed/providers/Microsoft.Compute/virtualMachines/v6y0n4l6q1w8i3p-np-delete-main-p2bwr-dv274 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v6y0n4l6q1w8i3p-np-delete-main-p2bwr-dv274: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-4vhtkz65lcdf--managed/providers/Microsoft.Compute/virtualMachines/v6y0n4l6q1w8i3p-np-delete-main-p2bwr-dv274 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-4t7wr4mxhmz6\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7261,"pr_state":"closed","pr_sha":"eb16c2e94e7f2596a478e5570733fb66ffcca02f","final_merged_sha":"f6667070546b139bd53a7df7d7674fd0c4250636","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T06:47:34Z","started_at":"2026-09-30T06:47:34Z","completed_at":"2026-09-30T10:35:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T10:45:58.664659857Z","timing_metadata_checked_at":"2026-09-30T10:45:58.664659857Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T10:45:58.667841546Z","region_metadata_checked_at":"2026-09-30T10:45:58.667841546Z"},"failed_at":["alert","e2e"],"failed_test_count":35,"occurrences":[{"row_id":"0db939d85773dffea3b3d8a6da5d5473aad64b08aa88cdb877774f3eb5d9d230","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"600a0314fbb714dba09f339df24ce54ab67395260e920baf7aff2d02983e4161","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014003d8\u003e: \n    resource group \"admin-credential-lifecycle-test-5rfj4vwnhrwj\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-vxghsm\" in resourcegroup=\"admin-credential-lifecycle-test-5rfj4vwnhrwj\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0efef022-0122-4549-9e52-3393af9918c1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0efef022-0122-4549-9e52-3393af9918c1\",\n      \"name\": \"0efef022-0122-4549-9e52-3393af9918c1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:44:51.882854873Z\",\n      \"endTime\": \"2026-09-30T08:08:52.436086515Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ofgpftr9snm78jc1alvisvlihdkq still exists (deletion dispatched at 2026-09-30T07:45:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014003d8\u003e: \n    resource group \"admin-credential-lifecycle-test-5rfj4vwnhrwj\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-vxghsm\" in resourcegroup=\"admin-credential-lifecycle-test-5rfj4vwnhrwj\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0efef022-0122-4549-9e52-3393af9918c1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0efef022-0122-4549-9e52-3393af9918c1\",\n      \"name\": \"0efef022-0122-4549-9e52-3393af9918c1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:44:51.882854873Z\",\n      \"endTime\": \"2026-09-30T08:08:52.436086515Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ofgpftr9snm78jc1alvisvlihdkq still exists (deletion dispatched at 2026-09-30T07:45:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1912ef3c9abb82e056d4056b8207722c9fd167a57f3f29e5a842e8be8f901bfd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"e8ae7ca73bac34509929f98cbc2e291873367df09e30b7ee513a70980408c7e4","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0009f4378\u003e: \n    resource group \"v20261001preview-glc4g8rxxg96\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-glc4g8rxxg96\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/22941031-f785-42ec-930f-02f9b4baf5df\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/22941031-f785-42ec-930f-02f9b4baf5df\",\n      \"name\": \"22941031-f785-42ec-930f-02f9b4baf5df\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:50:24.203578528Z\",\n      \"endTime\": \"2026-09-30T08:14:32.552450638Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56obpih7ob1nmt9oif2arpkqr6ctja still exists (deletion dispatched at 2026-09-30T07:50:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0009f4378\u003e: \n    resource group \"v20261001preview-glc4g8rxxg96\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-glc4g8rxxg96\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/22941031-f785-42ec-930f-02f9b4baf5df\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/22941031-f785-42ec-930f-02f9b4baf5df\",\n      \"name\": \"22941031-f785-42ec-930f-02f9b4baf5df\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:50:24.203578528Z\",\n      \"endTime\": \"2026-09-30T08:14:32.552450638Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56obpih7ob1nmt9oif2arpkqr6ctja still exists (deletion dispatched at 2026-09-30T07:50:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"19bd4c954029e6d57b43b641a1d2597db83c717994e1f1284f8e6a0ea5355ff1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"c425dbb349c3cb4e0d438e7efa64caf05e63c738219783e383f7f9abd3c127ca","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001e06228\u003e: \n    resource group \"rg-np-version-upgrade-lhpwgz-8wlwbkbk8nl7\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-lhpwgz\" in resourcegroup=\"rg-np-version-upgrade-lhpwgz-8wlwbkbk8nl7\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/32343bad-f62c-41f1-a78a-34ece808734d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/32343bad-f62c-41f1-a78a-34ece808734d\",\n      \"name\": \"32343bad-f62c-41f1-a78a-34ece808734d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:09:00.777952873Z\",\n      \"endTime\": \"2026-09-30T08:33:06.459889594Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56oj68ut6sfef5ef3g3kgp5llvpbpr still exists (deletion dispatched at 2026-09-30T08:09:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001e06228\u003e: \n    resource group \"rg-np-version-upgrade-lhpwgz-8wlwbkbk8nl7\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-lhpwgz\" in resourcegroup=\"rg-np-version-upgrade-lhpwgz-8wlwbkbk8nl7\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/32343bad-f62c-41f1-a78a-34ece808734d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/32343bad-f62c-41f1-a78a-34ece808734d\",\n      \"name\": \"32343bad-f62c-41f1-a78a-34ece808734d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:09:00.777952873Z\",\n      \"endTime\": \"2026-09-30T08:33:06.459889594Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56oj68ut6sfef5ef3g3kgp5llvpbpr still exists (deletion dispatched at 2026-09-30T08:09:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2119355726f4cf32f3822627de96caa5ededfa4274abf0e585a01a6b30ac492b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"a063820c19d1cfffec5676eb083b22917baff3755e10e5f7ba1e4ca7f468b87e","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c0e078\u003e: \n    resource group \"admin-api-breakglass-bvjprckvbd9j\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-bvjprckvbd9j\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/caa65f31-7834-40ae-9558-2b145d3d3707\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/caa65f31-7834-40ae-9558-2b145d3d3707\",\n      \"name\": \"caa65f31-7834-40ae-9558-2b145d3d3707\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:42:39.486123591Z\",\n      \"endTime\": \"2026-09-30T08:06:42.572288271Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56oguqit52ovjjoaakam9v60fven2c still exists (deletion dispatched at 2026-09-30T07:42:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c0e078\u003e: \n    resource group \"admin-api-breakglass-bvjprckvbd9j\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-bvjprckvbd9j\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/caa65f31-7834-40ae-9558-2b145d3d3707\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/caa65f31-7834-40ae-9558-2b145d3d3707\",\n      \"name\": \"caa65f31-7834-40ae-9558-2b145d3d3707\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:42:39.486123591Z\",\n      \"endTime\": \"2026-09-30T08:06:42.572288271Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56oguqit52ovjjoaakam9v60fven2c still exists (deletion dispatched at 2026-09-30T07:42:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"29b4cfd2569daa9c3d146129fdf2fc655434231fff8fc418d03d763d187358fd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"4990898d8a8d8d7b3cccfd436135763eddfa8bf40bd6a2221cf0582f059cda21","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e84138\u003e: \n    resource group \"rg-zstream-upgrade-4-21-d6bd8lqnx9k6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-mpqc9c\" in resourcegroup=\"rg-zstream-upgrade-4-21-d6bd8lqnx9k6\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9d0d023-161e-4fae-bc07-fa4aac05780f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9d0d023-161e-4fae-bc07-fa4aac05780f\",\n      \"name\": \"f9d0d023-161e-4fae-bc07-fa4aac05780f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:56:48.455291699Z\",\n      \"endTime\": \"2026-09-30T08:20:52.406066017Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ofmkuvbuga9h9ge40bfgpgl5gvfp still exists (deletion dispatched at 2026-09-30T07:57:02Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e84138\u003e: \n    resource group \"rg-zstream-upgrade-4-21-d6bd8lqnx9k6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-mpqc9c\" in resourcegroup=\"rg-zstream-upgrade-4-21-d6bd8lqnx9k6\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9d0d023-161e-4fae-bc07-fa4aac05780f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9d0d023-161e-4fae-bc07-fa4aac05780f\",\n      \"name\": \"f9d0d023-161e-4fae-bc07-fa4aac05780f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:56:48.455291699Z\",\n      \"endTime\": \"2026-09-30T08:20:52.406066017Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ofmkuvbuga9h9ge40bfgpgl5gvfp still exists (deletion dispatched at 2026-09-30T07:57:02Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2d105cebe60ed6812e94fadbef661524145d3c21ce52d8c0b49709a521e72963","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"afe091df3b6537610ba7d13c6ce02b998c9878ab1c5f2da66b17f37fa47b59c2","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00156e408\u003e: \n    resource group \"private-ingress-wkhlpqct8tlb\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-ingress\" in resourcegroup=\"private-ingress-wkhlpqct8tlb\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9cccb3e-9d83-49b8-9aa6-d3a585bd59e7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9cccb3e-9d83-49b8-9aa6-d3a585bd59e7\",\n      \"name\": \"a9cccb3e-9d83-49b8-9aa6-d3a585bd59e7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:47:33.546137807Z\",\n      \"endTime\": \"2026-09-30T08:11:42.306304509Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56oboi9api106r8emc4efcckh95di9 still exists (deletion dispatched at 2026-09-30T07:47:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00156e408\u003e: \n    resource group \"private-ingress-wkhlpqct8tlb\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-ingress\" in resourcegroup=\"private-ingress-wkhlpqct8tlb\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9cccb3e-9d83-49b8-9aa6-d3a585bd59e7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9cccb3e-9d83-49b8-9aa6-d3a585bd59e7\",\n      \"name\": \"a9cccb3e-9d83-49b8-9aa6-d3a585bd59e7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:47:33.546137807Z\",\n      \"endTime\": \"2026-09-30T08:11:42.306304509Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56oboi9api106r8emc4efcckh95di9 still exists (deletion dispatched at 2026-09-30T07:47:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"457c2f9baaabbe672ba588083f8d07313b8a27d17ef5c1f745fa7ee5a4ff568f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"61d2f9db9ee4e5c8023a43eed001242fe66442b4bd834571d5f4ea2180946254","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011481e0\u003e: \n    resource group \"rg-cluster-back-version-cnrv4422rs76\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-cnrv4422rs76\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-cnrv4422rs76\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-cnrv4422rs76, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-cnrv4422rs76\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-cnrv4422rs76, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011481e0\u003e: \n    resource group \"rg-cluster-back-version-cnrv4422rs76\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-cnrv4422rs76\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-cnrv4422rs76\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-cnrv4422rs76, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-cnrv4422rs76\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-cnrv4422rs76, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"293006e7a1ad208844fa10f4e2b3c0c8959662e799277a9c1d23e6534078c9e0","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"resource group \"rg-cluster-back-version-cnrv4422rs76\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-cnrv4422rs76\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-cnrv4422rs76\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-cnrv4422rs76, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-cnrv4422rs76\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-cnrv4422rs76, error: context deadline exceeded"}},{"row_id":"4710b0b2c0bbbb32df126dd783e1defe83e010505e6607bf334dab28576a4aa1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"f7618045d7cfaab4bd66f07a97cf823014f39a37fe0a4c093e781b82a7306d83","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00095ea98\u003e: \n    resource group \"ephemeral-osdisk-zwnb5bhrdzqz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-zwnb5bhrdzqz\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c5b0070-844b-4562-99f3-47e51a6c15ef\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c5b0070-844b-4562-99f3-47e51a6c15ef\",\n      \"name\": \"9c5b0070-844b-4562-99f3-47e51a6c15ef\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:45:01.77044017Z\",\n      \"endTime\": \"2026-09-30T08:09:02.54290012Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ok2d2baqjscldo4vmovim4i18pgh still exists (deletion dispatched at 2026-09-30T07:45:04Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00095ea98\u003e: \n    resource group \"ephemeral-osdisk-zwnb5bhrdzqz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-zwnb5bhrdzqz\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c5b0070-844b-4562-99f3-47e51a6c15ef\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c5b0070-844b-4562-99f3-47e51a6c15ef\",\n      \"name\": \"9c5b0070-844b-4562-99f3-47e51a6c15ef\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:45:01.77044017Z\",\n      \"endTime\": \"2026-09-30T08:09:02.54290012Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ok2d2baqjscldo4vmovim4i18pgh still exists (deletion dispatched at 2026-09-30T07:45:04Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4df2049e8ef6569c78e3875a8eedf4e39b331a6a7d0ea8f9c08af8d47d2d492d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"417b6f53e945c7f505f4ed684b6bc1182432048a48254ebedbdc5118b22e4c97","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001344078\u003e: \n    resource group \"rg-cp-ystream-upgrade-5-0-fs5gn7c5tqxf\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-56tt5z\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-fs5gn7c5tqxf\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52d0c5fd-02ac-47eb-a5f2-62e5ca81cff9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52d0c5fd-02ac-47eb-a5f2-62e5ca81cff9\",\n      \"name\": \"52d0c5fd-02ac-47eb-a5f2-62e5ca81cff9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:59:56.020744554Z\",\n      \"endTime\": \"2026-09-30T08:24:02.403347508Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56odktt8bp0otlfbosu69ogttm3nlc still exists (deletion dispatched at 2026-09-30T07:59:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001344078\u003e: \n    resource group \"rg-cp-ystream-upgrade-5-0-fs5gn7c5tqxf\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-56tt5z\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-fs5gn7c5tqxf\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52d0c5fd-02ac-47eb-a5f2-62e5ca81cff9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52d0c5fd-02ac-47eb-a5f2-62e5ca81cff9\",\n      \"name\": \"52d0c5fd-02ac-47eb-a5f2-62e5ca81cff9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:59:56.020744554Z\",\n      \"endTime\": \"2026-09-30T08:24:02.403347508Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56odktt8bp0otlfbosu69ogttm3nlc still exists (deletion dispatched at 2026-09-30T07:59:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"595ec19c6f6b5ff6a4579877bdc7fa2530aa28fa16fb6bf7e6e3937eae6902cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"alert","signature_id":"f4000e677b431fd645fc9061bdd557db73077acbe3281573024aff7d3154db6e","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T09:55:42Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j0283264-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-84tpd\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-84tpd (namespace arobit) on cluster ci01-j0283264-mgmt-2 has averaged 123.4% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"5a0953dc593a812609375791f8764824206d037c2bcae1ae50a10f82321d99e5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"ffa29da8af83e7964daeb9af651866d04306dbb88818e3e1019cbb1d504218e6","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00106a9d8\u003e: \n    resource group \"pullsecret-test-7dnj5wrv5xms\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7dnj5wrv5xms\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c782daed-b306-467e-81e3-8b6dd06ad52a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c782daed-b306-467e-81e3-8b6dd06ad52a\",\n      \"name\": \"c782daed-b306-467e-81e3-8b6dd06ad52a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:49:52.027158436Z\",\n      \"endTime\": \"2026-09-30T08:13:52.469740968Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ofor9qrgni6pa0caelkcpsahuun0 still exists (deletion dispatched at 2026-09-30T07:49:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00106a9d8\u003e: \n    resource group \"pullsecret-test-7dnj5wrv5xms\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7dnj5wrv5xms\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c782daed-b306-467e-81e3-8b6dd06ad52a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c782daed-b306-467e-81e3-8b6dd06ad52a\",\n      \"name\": \"c782daed-b306-467e-81e3-8b6dd06ad52a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:49:52.027158436Z\",\n      \"endTime\": \"2026-09-30T08:13:52.469740968Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ofor9qrgni6pa0caelkcpsahuun0 still exists (deletion dispatched at 2026-09-30T07:49:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5f3a2da81128ee3d309a1a135e4ec780d489500546e49753ac7e25044751d3b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"c635c7f3ad43e5d41b74812d30f60963314806e81ba8a3813c7a319c4b22a212","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001b6e0a8\u003e: \n    resource group \"customer-rg-hd4bhsrz7d7n\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-hd4bhsrz7d7n\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b56195ce-49cf-4cb1-b7c9-571ff13499d0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b56195ce-49cf-4cb1-b7c9-571ff13499d0\",\n      \"name\": \"b56195ce-49cf-4cb1-b7c9-571ff13499d0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:41:31.959909446Z\",\n      \"endTime\": \"2026-09-30T08:05:32.486384816Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56onftd04005nl721veuiicq8a5s6u still exists (deletion dispatched at 2026-09-30T07:41:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001b6e0a8\u003e: \n    resource group \"customer-rg-hd4bhsrz7d7n\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-hd4bhsrz7d7n\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b56195ce-49cf-4cb1-b7c9-571ff13499d0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b56195ce-49cf-4cb1-b7c9-571ff13499d0\",\n      \"name\": \"b56195ce-49cf-4cb1-b7c9-571ff13499d0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:41:31.959909446Z\",\n      \"endTime\": \"2026-09-30T08:05:32.486384816Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56onftd04005nl721veuiicq8a5s6u still exists (deletion dispatched at 2026-09-30T07:41:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"636a032946e462d2a6e4384a327344addbea1c3f9e8e70631735f2d029efe071","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"1922b6404a193a0248ac9de20c3029fc58e5a940536058fc0aa8443f32082444","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001baa138\u003e: \n    resource group \"np-autoscaling-x6vzkczm54sv\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-x6vzkczm54sv\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/be95d509-ef21-4815-a213-b7566866b60c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/be95d509-ef21-4815-a213-b7566866b60c\",\n      \"name\": \"be95d509-ef21-4815-a213-b7566866b60c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:57:15.728931938Z\",\n      \"endTime\": \"2026-09-30T08:21:22.412329675Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56oig3m7l5t041s2h3535h24baigrd still exists (deletion dispatched at 2026-09-30T07:57:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001baa138\u003e: \n    resource group \"np-autoscaling-x6vzkczm54sv\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-x6vzkczm54sv\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/be95d509-ef21-4815-a213-b7566866b60c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/be95d509-ef21-4815-a213-b7566866b60c\",\n      \"name\": \"be95d509-ef21-4815-a213-b7566866b60c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:57:15.728931938Z\",\n      \"endTime\": \"2026-09-30T08:21:22.412329675Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56oig3m7l5t041s2h3535h24baigrd still exists (deletion dispatched at 2026-09-30T07:57:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"65690483a5d0925dcd7b439367e4c9415af693973b4f0d523e3872a97a976866","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"9825bceafad7b29cf6ff082816b91e1a07777df23ef630f76b47894fe3017f42","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc002180138\u003e: \n    resource group \"feature-aggregation-7452j745q24m\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-7452j745q24m\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a344fed7-604f-4718-a10f-cca3aa460a69\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a344fed7-604f-4718-a10f-cca3aa460a69\",\n      \"name\": \"a344fed7-604f-4718-a10f-cca3aa460a69\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:03:38.750208141Z\",\n      \"endTime\": \"2026-09-30T08:27:42.421715531Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56pklkbe3f9k9tk3gemfqinnl5654u still exists (deletion dispatched at 2026-09-30T08:03:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc002180138\u003e: \n    resource group \"feature-aggregation-7452j745q24m\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-7452j745q24m\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a344fed7-604f-4718-a10f-cca3aa460a69\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a344fed7-604f-4718-a10f-cca3aa460a69\",\n      \"name\": \"a344fed7-604f-4718-a10f-cca3aa460a69\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:03:38.750208141Z\",\n      \"endTime\": \"2026-09-30T08:27:42.421715531Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56pklkbe3f9k9tk3gemfqinnl5654u still exists (deletion dispatched at 2026-09-30T08:03:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"711603b0733fa1910389f77e88ed94da115ca485187912d5966d895609fa454a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"397097b128fb1f71d9cac290dbcc3acd3e5a33bf08922e38de2034ce0a98cded","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0018c61c8\u003e: \n    resource group \"cni-cilium-5mwkmtbx9lm6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-5mwkmtbx9lm6\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc69ed49-5ecc-40ea-9a0d-84c38d022d57\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc69ed49-5ecc-40ea-9a0d-84c38d022d57\",\n      \"name\": \"fc69ed49-5ecc-40ea-9a0d-84c38d022d57\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:47:13.892958657Z\",\n      \"endTime\": \"2026-09-30T08:11:22.315778755Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ojjtudpbnkdj9hn5lbovlb6qujk0 still exists (deletion dispatched at 2026-09-30T07:47:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0018c61c8\u003e: \n    resource group \"cni-cilium-5mwkmtbx9lm6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-5mwkmtbx9lm6\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc69ed49-5ecc-40ea-9a0d-84c38d022d57\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc69ed49-5ecc-40ea-9a0d-84c38d022d57\",\n      \"name\": \"fc69ed49-5ecc-40ea-9a0d-84c38d022d57\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:47:13.892958657Z\",\n      \"endTime\": \"2026-09-30T08:11:22.315778755Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ojjtudpbnkdj9hn5lbovlb6qujk0 still exists (deletion dispatched at 2026-09-30T07:47:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"71787115394623ea33a3c29b5233794079ade531f8f1a6938055a3fc0f512346","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"649e383ec3ed1073212333a1b5e629c51797a17f0420563d94bcc96852af6d6a","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0016d82b8\u003e: \n    resource group \"admin-api-serialconsole-bootdiag-bztnlzgvbtfg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-bztnlzgvbtfg\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/abff150f-53b5-4a1e-8e79-85cb6932d14c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/abff150f-53b5-4a1e-8e79-85cb6932d14c\",\n      \"name\": \"abff150f-53b5-4a1e-8e79-85cb6932d14c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:44:47.122956765Z\",\n      \"endTime\": \"2026-09-30T08:08:52.492586002Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56oclgsj9oapvabnc01b65dqdln9hs still exists (deletion dispatched at 2026-09-30T07:44:55Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0016d82b8\u003e: \n    resource group \"admin-api-serialconsole-bootdiag-bztnlzgvbtfg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-bztnlzgvbtfg\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/abff150f-53b5-4a1e-8e79-85cb6932d14c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/abff150f-53b5-4a1e-8e79-85cb6932d14c\",\n      \"name\": \"abff150f-53b5-4a1e-8e79-85cb6932d14c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:44:47.122956765Z\",\n      \"endTime\": \"2026-09-30T08:08:52.492586002Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56oclgsj9oapvabnc01b65dqdln9hs still exists (deletion dispatched at 2026-09-30T07:44:55Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"83c3d1f97626231197f625d05f64e3ad2ffd19a8b3e3e250667f61c9a171e495","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"bb57716abde8e2e547ac440d3603b828a0aa183bfdf756ba7db290fe0da391f6","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f6c198\u003e: \n    resource group \"private-kas-htntrcf5948s\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-htntrcf5948s\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f1e211cf-72b5-460c-86b9-42f8c0ab026d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f1e211cf-72b5-460c-86b9-42f8c0ab026d\",\n      \"name\": \"f1e211cf-72b5-460c-86b9-42f8c0ab026d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:53:38.631153455Z\",\n      \"endTime\": \"2026-09-30T08:17:42.452924542Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ofmlsp87gpoiuva34g4ukacv58an still exists (deletion dispatched at 2026-09-30T07:53:57Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f6c198\u003e: \n    resource group \"private-kas-htntrcf5948s\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-htntrcf5948s\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f1e211cf-72b5-460c-86b9-42f8c0ab026d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f1e211cf-72b5-460c-86b9-42f8c0ab026d\",\n      \"name\": \"f1e211cf-72b5-460c-86b9-42f8c0ab026d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:53:38.631153455Z\",\n      \"endTime\": \"2026-09-30T08:17:42.452924542Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ofmlsp87gpoiuva34g4ukacv58an still exists (deletion dispatched at 2026-09-30T07:53:57Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"84e937aef87277029c42701d5c8639bd9a7d644c57a594394930765671e9d0fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"100fa41546dbaca127b035a4f15ea05b9f9ef7056f944eee271ee12155761b27","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001190090\u003e: \n    resource group \"rg-zstream-upgrade-5-0-vztfjbhwj86s\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-5-0-dbc7n2\" in resourcegroup=\"rg-zstream-upgrade-5-0-vztfjbhwj86s\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86d21c47-4deb-485c-adc0-8dba2db04e85\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86d21c47-4deb-485c-adc0-8dba2db04e85\",\n      \"name\": \"86d21c47-4deb-485c-adc0-8dba2db04e85\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:00:45.311282131Z\",\n      \"endTime\": \"2026-09-30T08:24:52.368269792Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ojo9ccai43f073bp4dpipm3b14ma still exists (deletion dispatched at 2026-09-30T08:01:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001190090\u003e: \n    resource group \"rg-zstream-upgrade-5-0-vztfjbhwj86s\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-5-0-dbc7n2\" in resourcegroup=\"rg-zstream-upgrade-5-0-vztfjbhwj86s\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86d21c47-4deb-485c-adc0-8dba2db04e85\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86d21c47-4deb-485c-adc0-8dba2db04e85\",\n      \"name\": \"86d21c47-4deb-485c-adc0-8dba2db04e85\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:00:45.311282131Z\",\n      \"endTime\": \"2026-09-30T08:24:52.368269792Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ojo9ccai43f073bp4dpipm3b14ma still exists (deletion dispatched at 2026-09-30T08:01:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8a922694c8efe1074ae2aab65bd190ca080ec64f4ffcc72fca76326b405c8994","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"alert","signature_id":"3850893a2be376a4c082764baaa2865ab6202df69d12b07798777716859179b2","test_name":"[aro-hcp-observability] [hcp] alert HCPClusterOperatorUnavailable does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T09:10:27Z\n  Ended:       2026-09-30T10:12:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t56ofmlsp87gpoiuva34g4ukacv58an-private-kas\", region=\"westus3\", severity=\"warning\"\n  Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2t56ofmlsp87gpoiuva34g4ukacv58an-private-kas (management cluster ci01-j0283264-mgmt-2) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c65eaecf2bcf9cab816e9e9171a4f334676466d8dfb2f88bb604984fc8e6cfe2","failure_pattern":"alert [hcp] HCPClusterOperatorUnavailable fired","search_query":"alert [hcp] HCPClusterOperatorUnavailable fired"}},{"row_id":"8fa4a6f00e1055bf61cd1a7994677e75d1b7029d0ae06f8b985bc51382a8ea37","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"765cb31fbe9731b40edd3678a0fe26d7a83e359d9cc7b2b09560b2f8ec5e71aa","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:409 +0x87 fp=0xc0000d9e90 sp=0xc0000d9e20 pc=0xbff15a7\ngithub.com/Azure/ARO-HCP/test/util/framework.runResourceGroupCleanup.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:410 +0x7e fp=0xc0000d9f78 sp=0xc0000d9e90 pc=0xbff12be\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:93 +0x50 fp=0xc0000d9fe0 sp=0xc0000d9f78 pc=0x6f51610\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000d9fe8 sp=0xc0000d9fe0 pc=0x5439fc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 3248\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 3283 gp=0xc0011bfdc0 m=nil [select, 2 minutes]:\nruntime.gopark(0xc000e65f48?, 0x2?, 0xc0?, 0x5d?, 0xc000e65ef4?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000df4d78 sp=0xc000df4d58 pc=0x54317ae\nruntime.selectgo(0xc000df4f48, 0xc000e65ef0, 0xc00191c0c0?, 0x0, 0xc0028017d0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000df4eb8 sp=0xc000df4d78 pc=0x540c3b7\nnet/http.(*persistConn).writeLoop(0xc0013e3b00)\n\t/usr/lib/golang/src/net/http/transport.go:2600 +0xe7 fp=0xc000df4fc8 sp=0xc000df4eb8 pc=0x5803227\nnet/http.(*Transport).dialConn.gowrap3()\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x25 fp=0xc000df4fe0 sp=0xc000df4fc8 pc=0x57ffec5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000df4fe8 sp=0xc000df4fe0 pc=0x5439fc1\ncreated by net/http.(*Transport).dialConn in goroutine 3249\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x17a5\n\ngoroutine 3232 gp=0xc0002616c0 m=nil [select]:\nruntime.gopark(0xc001a7f768?, 0x2?, 0x40?, 0x0?, 0xc001a7f75c?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc001a7f5e8 sp=0xc001a7f5c8 pc=0x54317ae\nruntime.selectgo(0xc001a7f768, 0xc001a7f758, 0xc001a7f788?, 0x0, 0xf83b8ce?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc001a7f728 sp=0xc001a7f5e8 pc=0x540c3b7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x108bfe40, 0xc00066ad20}, 0xf7fc714?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.23.0/internal/shared/shared.go:36 +0x76 fp=0xc001a7f798 sp=0xc001a7f728 pc=0x5c1c996\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x10919ee0, {0x108bfe40, 0xc00066ad20}, 0x542ed93)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.23.0/runtime/poller.go:284 +0x732 fp=0xc001a7fa98 sp=0xc001a7f798 pc=0x5e69a92\ngithub.com/Azure/ARO-HCP/test/util/framework.deleteHCPClusterAttempt({0x108bfe40, 0xc00066ad20}, 0xc00104bae8, {0xc000f74aa0, 0x1c}, {0xc0018c1ef0, 0x15})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/helpers_v20240610preview.go:591 +0x365 fp=0xc001a7fc20 sp=0xc001a7fa98 pc=0xbfaf265\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610.func1({0x108bfe40?, 0xc00066ad20?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/helpers_v20240610preview.go:628 +0x72 fp=0xc001a7fcc8 sp=0xc001a7fc20 pc=0xbfafa12\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext({0x108bfe40?, 0xc00066ad20?}, 0xc00066ad30?)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:159 +0x62 fp=0xc001a7fd20 sp=0xc001a7fcc8 pc=0x5bd1a22\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoffWithContext({0x108bfe40, 0xc00066ad20}, {0xdf8475800, 0x4000000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0xc001a7fe68)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:501 +0xae fp=0xc001a7fd98 sp=0xc001a7fd20 pc=0x5bcface\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610({0x108bfdd0, 0xc000f19220}, 0xc00104bae8, {0xc000f74aa0, 0x1c}, {0xc0018c1ef0, 0x15}, 0x15d3ef79800)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/helpers_v20240610preview.go:626 +0x285 fp=0xc001a7fef0 sp=0xc001a7fd98 pc=0xbfaf785\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/helpers_v20240610preview.go:799 +0x7e fp=0xc001a7ff78 sp=0xc001a7fef0 pc=0xbfb219e\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:93 +0x50 fp=0xc001a7ffe0 sp=0xc001a7ff78 pc=0x6f51610\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc001a7ffe8 sp=0xc001a7ffe0 pc=0x5439fc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 3273\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:78 +0x95\n\nrax    0xca\nrbx    0x0\nrcx    0x543bde3\nrdx    0x0\nrdi    0x169679d0\nrsi    0x80\nrbp    0x7ffc297c0df0\nrsp    0x7ffc297c0da8\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x3\nr13    0x1\nr14    0x16965440\nr15    0x5\nrip    0x543bde1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\napp-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:409 +0x87 fp=0xc0000d9e90 sp=0xc0000d9e20 pc=0xbff15a7\ngithub.com/Azure/ARO-HCP/test/util/framework.runResourceGroupCleanup.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:410 +0x7e fp=0xc0000d9f78 sp=0xc0000d9e90 pc=0xbff12be\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:93 +0x50 fp=0xc0000d9fe0 sp=0xc0000d9f78 pc=0x6f51610\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000d9fe8 sp=0xc0000d9fe0 pc=0x5439fc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 3248\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 3283 gp=0xc0011bfdc0 m=nil [select, 2 minutes]:\nruntime.gopark(0xc000e65f48?, 0x2?, 0xc0?, 0x5d?, 0xc000e65ef4?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000df4d78 sp=0xc000df4d58 pc=0x54317ae\nruntime.selectgo(0xc000df4f48, 0xc000e65ef0, 0xc00191c0c0?, 0x0, 0xc0028017d0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000df4eb8 sp=0xc000df4d78 pc=0x540c3b7\nnet/http.(*persistConn).writeLoop(0xc0013e3b00)\n\t/usr/lib/golang/src/net/http/transport.go:2600 +0xe7 fp=0xc000df4fc8 sp=0xc000df4eb8 pc=0x5803227\nnet/http.(*Transport).dialConn.gowrap3()\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x25 fp=0xc000df4fe0 sp=0xc000df4fc8 pc=0x57ffec5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000df4fe8 sp=0xc000df4fe0 pc=0x5439fc1\ncreated by net/http.(*Transport).dialConn in goroutine 3249\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x17a5\n\ngoroutine 3232 gp=0xc0002616c0 m=nil [select]:\nruntime.gopark(0xc001a7f768?, 0x2?, 0x40?, 0x0?, 0xc001a7f75c?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc001a7f5e8 sp=0xc001a7f5c8 pc=0x54317ae\nruntime.selectgo(0xc001a7f768, 0xc001a7f758, 0xc001a7f788?, 0x0, 0xf83b8ce?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc001a7f728 sp=0xc001a7f5e8 pc=0x540c3b7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x108bfe40, 0xc00066ad20}, 0xf7fc714?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.23.0/internal/shared/shared.go:36 +0x76 fp=0xc001a7f798 sp=0xc001a7f728 pc=0x5c1c996\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x10919ee0, {0x108bfe40, 0xc00066ad20}, 0x542ed93)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.23.0/runtime/poller.go:284 +0x732 fp=0xc001a7fa98 sp=0xc001a7f798 pc=0x5e69a92\ngithub.com/Azure/ARO-HCP/test/util/framework.deleteHCPClusterAttempt({0x108bfe40, 0xc00066ad20}, 0xc00104bae8, {0xc000f74aa0, 0x1c}, {0xc0018c1ef0, 0x15})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/helpers_v20240610preview.go:591 +0x365 fp=0xc001a7fc20 sp=0xc001a7fa98 pc=0xbfaf265\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610.func1({0x108bfe40?, 0xc00066ad20?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/helpers_v20240610preview.go:628 +0x72 fp=0xc001a7fcc8 sp=0xc001a7fc20 pc=0xbfafa12\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext({0x108bfe40?, 0xc00066ad20?}, 0xc00066ad30?)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:159 +0x62 fp=0xc001a7fd20 sp=0xc001a7fcc8 pc=0x5bd1a22\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoffWithContext({0x108bfe40, 0xc00066ad20}, {0xdf8475800, 0x4000000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0xc001a7fe68)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:501 +0xae fp=0xc001a7fd98 sp=0xc001a7fd20 pc=0x5bcface\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610({0x108bfdd0, 0xc000f19220}, 0xc00104bae8, {0xc000f74aa0, 0x1c}, {0xc0018c1ef0, 0x15}, 0x15d3ef79800)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/helpers_v20240610preview.go:626 +0x285 fp=0xc001a7fef0 sp=0xc001a7fd98 pc=0xbfaf785\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/helpers_v20240610preview.go:799 +0x7e fp=0xc001a7ff78 sp=0xc001a7fef0 pc=0xbfb219e\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:93 +0x50 fp=0xc001a7ffe0 sp=0xc001a7ff78 pc=0x6f51610\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc001a7ffe8 sp=0xc001a7ffe0 pc=0x5439fc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 3273\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:78 +0x95\n\nrax    0xca\nrbx    0x0\nrcx    0x543bde3\nrdx    0x0\nrdi    0x169679d0\nrsi    0x80\nrbp    0x7ffc297c0df0\nrsp    0x7ffc297c0da8\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x3\nr13    0x1\nr14    0x16965440\nr15    0x5\nrip    0x543bde1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"392750d533ddeabf76d5209f9e2a09e5f48e64ee98fce8c13f1c6bd36af29cfa","failure_pattern":"Deserialization Error: no output from command","search_query":"Deserialization Error: no output from command"}},{"row_id":"92c5d4ff8e65eb56ab451a54560d9e78da13db079747c61455c2666cc78d0dc1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"ec90d4af2927302761e0ebd2e1983af136cd63955ff9ce58162405d9f1e62c70","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011781c8\u003e: \n    resource group \"idms-6clt25jfd748\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-6clt25jfd748\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95e22e88-1144-4d00-8cc9-dc9f1a6c3ffb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95e22e88-1144-4d00-8cc9-dc9f1a6c3ffb\",\n      \"name\": \"95e22e88-1144-4d00-8cc9-dc9f1a6c3ffb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:43:50.391649068Z\",\n      \"endTime\": \"2026-09-30T08:07:52.519744698Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ofblporgmut8brildeurh8bqh0jm still exists (deletion dispatched at 2026-09-30T07:44:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011781c8\u003e: \n    resource group \"idms-6clt25jfd748\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-6clt25jfd748\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95e22e88-1144-4d00-8cc9-dc9f1a6c3ffb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95e22e88-1144-4d00-8cc9-dc9f1a6c3ffb\",\n      \"name\": \"95e22e88-1144-4d00-8cc9-dc9f1a6c3ffb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:43:50.391649068Z\",\n      \"endTime\": \"2026-09-30T08:07:52.519744698Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ofblporgmut8brildeurh8bqh0jm still exists (deletion dispatched at 2026-09-30T07:44:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a63d6d9e21a8784c60c9b1367f823beff83c1a3008f7bc7766943d7ddc6e6bf4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"00b952975e25b271682c84963b4e373b62836b85fd377a0a370a32c2e1265937","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001702090\u003e: \n    resource group \"fips-enabled-xs9fxbdgnwhz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-xs9fxbdgnwhz\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/81d3effe-1da4-43c5-8596-8a54252ca278\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/81d3effe-1da4-43c5-8596-8a54252ca278\",\n      \"name\": \"81d3effe-1da4-43c5-8596-8a54252ca278\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:45:23.642226584Z\",\n      \"endTime\": \"2026-09-30T08:09:32.307764104Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56obh88cddik6iio0cu4ci35j02k7f still exists (deletion dispatched at 2026-09-30T07:45:28Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001702090\u003e: \n    resource group \"fips-enabled-xs9fxbdgnwhz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-xs9fxbdgnwhz\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/81d3effe-1da4-43c5-8596-8a54252ca278\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/81d3effe-1da4-43c5-8596-8a54252ca278\",\n      \"name\": \"81d3effe-1da4-43c5-8596-8a54252ca278\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:45:23.642226584Z\",\n      \"endTime\": \"2026-09-30T08:09:32.307764104Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56obh88cddik6iio0cu4ci35j02k7f still exists (deletion dispatched at 2026-09-30T07:45:28Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a76da1600985c76f3b9d52aa3f285b7c1e8529df9ac813b7c1bce004ff5b8ab9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"7456f43c64352e2292d78935707ba348a1cd19ec7dd8562f66846dacd59a897b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000d94390\u003e: \n    resource group \"rg-candidate-4-22-7rr572k8p4nx\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-hqg9nr\" in resourcegroup=\"rg-candidate-4-22-7rr572k8p4nx\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5146bdc-a88e-43b4-ade8-4264fc68118b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5146bdc-a88e-43b4-ade8-4264fc68118b\",\n      \"name\": \"c5146bdc-a88e-43b4-ade8-4264fc68118b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:52:17.26155411Z\",\n      \"endTime\": \"2026-09-30T08:16:22.313143273Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ojq845tseueb8p9jb82rbbpq2mbb still exists (deletion dispatched at 2026-09-30T07:52:24Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000d94390\u003e: \n    resource group \"rg-candidate-4-22-7rr572k8p4nx\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-hqg9nr\" in resourcegroup=\"rg-candidate-4-22-7rr572k8p4nx\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5146bdc-a88e-43b4-ade8-4264fc68118b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5146bdc-a88e-43b4-ade8-4264fc68118b\",\n      \"name\": \"c5146bdc-a88e-43b4-ade8-4264fc68118b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:52:17.26155411Z\",\n      \"endTime\": \"2026-09-30T08:16:22.313143273Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ojq845tseueb8p9jb82rbbpq2mbb still exists (deletion dispatched at 2026-09-30T07:52:24Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a7877d69e35d62cecdfcd0dc96c267eac18b1aa045aee005c9023c988d2e9185","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"e4780041957a895982016aa7b2489f3353971e88275784b941e3e61d6674b776","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f38108\u003e: \n    resource group \"v20260901preview-fphhhngljlhp\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-fphhhngljlhp\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc213230-cecc-45fc-a51e-9966fc3f25a4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc213230-cecc-45fc-a51e-9966fc3f25a4\",\n      \"name\": \"dc213230-cecc-45fc-a51e-9966fc3f25a4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:49:49.418893525Z\",\n      \"endTime\": \"2026-09-30T08:13:52.389241283Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ocsot5cpua59vgr84hkknkr96gf6 still exists (deletion dispatched at 2026-09-30T07:49:51Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f38108\u003e: \n    resource group \"v20260901preview-fphhhngljlhp\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-fphhhngljlhp\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc213230-cecc-45fc-a51e-9966fc3f25a4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc213230-cecc-45fc-a51e-9966fc3f25a4\",\n      \"name\": \"dc213230-cecc-45fc-a51e-9966fc3f25a4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:49:49.418893525Z\",\n      \"endTime\": \"2026-09-30T08:13:52.389241283Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ocsot5cpua59vgr84hkknkr96gf6 still exists (deletion dispatched at 2026-09-30T07:49:51Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"afca035b66214935e7fe44a095fe520dae014c836b3128601d5cf587b8be842b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"7bf451011d3828ddbe210bbc04c6069d4b08635a16722de7eac25ed2398705ec","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001396090\u003e: \n    resource group \"rg-cp-ystream-upgrade-4-23-wm46s5gt65dl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-pg8svf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-wm46s5gt65dl\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4f33f8f6-72a9-44ff-8c84-6b45107fd2f9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4f33f8f6-72a9-44ff-8c84-6b45107fd2f9\",\n      \"name\": \"4f33f8f6-72a9-44ff-8c84-6b45107fd2f9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:57:46.367143777Z\",\n      \"endTime\": \"2026-09-30T08:21:52.387303059Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56odfbmlrdpspcs2jrvj7ok5eocb47 still exists (deletion dispatched at 2026-09-30T07:57:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001396090\u003e: \n    resource group \"rg-cp-ystream-upgrade-4-23-wm46s5gt65dl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-pg8svf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-wm46s5gt65dl\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4f33f8f6-72a9-44ff-8c84-6b45107fd2f9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4f33f8f6-72a9-44ff-8c84-6b45107fd2f9\",\n      \"name\": \"4f33f8f6-72a9-44ff-8c84-6b45107fd2f9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:57:46.367143777Z\",\n      \"endTime\": \"2026-09-30T08:21:52.387303059Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56odfbmlrdpspcs2jrvj7ok5eocb47 still exists (deletion dispatched at 2026-09-30T07:57:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b6eee3cc0e2a3ca0f213841293541de2a56f897907a0f151f8582adabb33c3ac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"alert","signature_id":"7a693951723740214ffa37293275209a959529bb537f41e30e9db140a02cecbd","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 16 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:36:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ofgpftr9snm78jc1alvisvlihdkq-z9j9b2e5q2j7y1b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56ofgpftr9snm78jc1alvisvlihdkq-z9j9b2e5q2j7y1b/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:36:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ofblporgmut8brildeurh8bqh0jm-b8a2i8y2x4y0w5h\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56ofblporgmut8brildeurh8bqh0jm-b8a2i8y2x4y0w5h/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:36:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56obh88cddik6iio0cu4ci35j02k7f-f1s2k3l8b2h6n5y\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56obh88cddik6iio0cu4ci35j02k7f-f1s2k3l8b2h6n5y/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:36:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56oguqit52ovjjoaakam9v60fven2c-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56oguqit52ovjjoaakam9v60fven2c-sre-hcp-cluster/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:36:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56oclgsj9oapvabnc01b65dqdln9hs-q5s4p4g3c9i1p1b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56oclgsj9oapvabnc01b65dqdln9hs-q5s4p4g3c9i1p1b/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:36:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56onftd04005nl721veuiicq8a5s6u-p1f3j3e2l1z4c9o\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56onftd04005nl721veuiicq8a5s6u-p1f3j3e2l1z4c9o/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:44:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56obh88cddik6iio0cu4ci35j02k7f-f1s2k3l8b2h6n5y\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56obh88cddik6iio0cu4ci35j02k7f-f1s2k3l8b2h6n5y/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:44:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ofblporgmut8brildeurh8bqh0jm-b8a2i8y2x4y0w5h\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56ofblporgmut8brildeurh8bqh0jm-b8a2i8y2x4y0w5h/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:44:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56oguqit52ovjjoaakam9v60fven2c-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56oguqit52ovjjoaakam9v60fven2c-sre-hcp-cluster/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:44:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56onftd04005nl721veuiicq8a5s6u-p1f3j3e2l1z4c9o\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56onftd04005nl721veuiicq8a5s6u-p1f3j3e2l1z4c9o/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:44:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ofgpftr9snm78jc1alvisvlihdkq-z9j9b2e5q2j7y1b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56ofgpftr9snm78jc1alvisvlihdkq-z9j9b2e5q2j7y1b/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:46:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56oclgsj9oapvabnc01b65dqdln9hs-q5s4p4g3c9i1p1b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56oclgsj9oapvabnc01b65dqdln9hs-q5s4p4g3c9i1p1b/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T09:10:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ok2d2baqjscldo4vmovim4i18pgh-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56ok2d2baqjscldo4vmovim4i18pgh-ephemeral-disk/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 14:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T09:10:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56p42at8ondug5mpb3mmh6f9hh71ck-ea-listrk7c29\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56p42at8ondug5mpb3mmh6f9hh71ck-ea-listrk7c29/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 15:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T09:10:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56p42at8ondug5mpb3mmh6f9hh71ck-ea-listrk7c29\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56p42at8ondug5mpb3mmh6f9hh71ck-ea-listrk7c29/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 16:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T09:10:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ok2d2baqjscldo4vmovim4i18pgh-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56ok2d2baqjscldo4vmovim4i18pgh-ephemeral-disk/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"ba89909be147865be314b6b374fbf189231f9019af30fb98657cb24dd2dfd553","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"a1e6e6a948237d5895cda86936bae09d2fd958c75bf3854eb19fbf06516789a2","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:301]: failed to update cluster with new KMS key with the second rotation\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000dfb3e0\u003e: \n    failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-vxcvln6bjmck\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-vxcvln6bjmck, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:301]: failed to update cluster with new KMS key with the second rotation\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000dfb3e0\u003e: \n    failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-vxcvln6bjmck\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-vxcvln6bjmck, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f570ff2ef05014779bb7489dca295e256a69aead33d6bff8319ed54db32fd9e1","failure_pattern":"UpdateHCPCluster20260901 timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-vxcvln6bjmck\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-vxcvln6bjmck, error: context deadline exceeded"}},{"row_id":"c0d2df83ea88a2c469daf30d1c69a1ccfb2c026abd14cb08e2615935c77108d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"5446c05e3ffbb0d0281fc86603f62bdf8f4380f4ed38b1703787e9b1478ab704","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019ec0d8\u003e: \n    resource group \"rg-zstream-upgrade-4-23-gq8tt4j82btd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-23-jjk68g\" in resourcegroup=\"rg-zstream-upgrade-4-23-gq8tt4j82btd\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/259e7251-93cc-4a54-a228-5ab61010d78f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/259e7251-93cc-4a54-a228-5ab61010d78f\",\n      \"name\": \"259e7251-93cc-4a54-a228-5ab61010d78f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:02:15.27366157Z\",\n      \"endTime\": \"2026-09-30T08:26:22.388447131Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ogva7br99jqa1217jecnq155d1ej still exists (deletion dispatched at 2026-09-30T08:02:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019ec0d8\u003e: \n    resource group \"rg-zstream-upgrade-4-23-gq8tt4j82btd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-23-jjk68g\" in resourcegroup=\"rg-zstream-upgrade-4-23-gq8tt4j82btd\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/259e7251-93cc-4a54-a228-5ab61010d78f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/259e7251-93cc-4a54-a228-5ab61010d78f\",\n      \"name\": \"259e7251-93cc-4a54-a228-5ab61010d78f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:02:15.27366157Z\",\n      \"endTime\": \"2026-09-30T08:26:22.388447131Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ogva7br99jqa1217jecnq155d1ej still exists (deletion dispatched at 2026-09-30T08:02:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cb0af13c1d3f394e303987bfff96c8c8cd887624d6a2e25d08c2d7e6e0eb0287","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"alert","signature_id":"ff1be5ecc56cbed8526bc0d055a5d690c6803444053e4d055eb54c8a29112a59","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T09:05:33Z\n  Ended:       2026-09-30T09:25:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j0283264-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.44:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/services-j0283264\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-8bc58bd6b-kmgbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"kubernetesevents\"\n  Description: Kusto log data for table kubernetesevents on cluster ci01-j0283264-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T09:06:32Z\n  Ended:       2026-09-30T09:26:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j0283264-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.44:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/services-j0283264\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-8bc58bd6b-kmgbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"kubernetesevents\"\n  Description: Kusto log data for table kubernetesevents on cluster ci01-j0283264-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","failure_pattern":"alert [svc] KustoLogsDataStale fired","search_query":"alert [svc] KustoLogsDataStale fired"}},{"row_id":"cd7ddf8a193262151390010254ef02d9d4a5a229be543633399c91f1fbf78570","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"1cfe895636607d50e1837e523cbd8ebfebbeb7a06dfe051208e969612f8d36ce","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010642a0\u003e: \n    resource group \"external-auth-cluster-4nlpdcdjkq8m\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-4nlpdcdjkq8m\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0f11d658-22eb-436f-a6d0-266fcfe0ad04\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0f11d658-22eb-436f-a6d0-266fcfe0ad04\",\n      \"name\": \"0f11d658-22eb-436f-a6d0-266fcfe0ad04\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:50:30.578221498Z\",\n      \"endTime\": \"2026-09-30T08:14:32.551976542Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56odm3uljbr1bc72mqo3pd7c5dcrar still exists (deletion dispatched at 2026-09-30T07:50:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010642a0\u003e: \n    resource group \"external-auth-cluster-4nlpdcdjkq8m\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-4nlpdcdjkq8m\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0f11d658-22eb-436f-a6d0-266fcfe0ad04\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0f11d658-22eb-436f-a6d0-266fcfe0ad04\",\n      \"name\": \"0f11d658-22eb-436f-a6d0-266fcfe0ad04\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:50:30.578221498Z\",\n      \"endTime\": \"2026-09-30T08:14:32.551976542Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56odm3uljbr1bc72mqo3pd7c5dcrar still exists (deletion dispatched at 2026-09-30T07:50:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d91756101f8b4172ed4dd117077600e0a6bd676b3c37b8f67fd945deac25a667","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"alert","signature_id":"00abdede94e48868df7e3ac1321fbb792324089549a037edfb3ae75e29326212","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 32 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:54:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ok2d2baqjscldo4vmovim4i18pgh-ephemeral-disk\", pod=\"cloud-network-config-controller-6b7699b59d-mtvjg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1aa0eaea-3879-4a75-b74f-75ef531c3b1d\"\n  Description: Pod ocm-arohcpci01-2t56ok2d2baqjscldo4vmovim4i18pgh-ephemeral-disk/cloud-network-config-controller-6b7699b59d-mtvjg (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:54:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56onftd04005nl721veuiicq8a5s6u-p1f3j3e2l1z4c9o\", pod=\"cloud-network-config-controller-c9f8f4988-98sd7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1b3a2719-6e7a-4a5b-aeae-d318e5e4d17e\"\n  Description: Pod ocm-arohcpci01-2t56onftd04005nl721veuiicq8a5s6u-p1f3j3e2l1z4c9o/cloud-network-config-controller-c9f8f4988-98sd7 (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:54:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56p42at8ondug5mpb3mmh6f9hh71ck-ea-listrk7c29\", pod=\"csi-snapshot-controller-ff4fdf4ff-8fn8s\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"523581cf-5012-43ac-a402-c7a0a8acaac4\"\n  Description: Pod ocm-arohcpci01-2t56p42at8ondug5mpb3mmh6f9hh71ck-ea-listrk7c29/csi-snapshot-controller-ff4fdf4ff-8fn8s (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:54:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56p42at8ondug5mpb3mmh6f9hh71ck-ea-listrk7c29\", pod=\"cloud-network-config-controller-5c94ccc4b5-cg2cp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"dafd5d14-e303-47e0-b4e1-d01b00ea97f1\"\n  Description: Pod ocm-arohcpci01-2t56p42at8ondug5mpb3mmh6f9hh71ck-ea-listrk7c29/cloud-network-config-controller-5c94ccc4b5-cg2cp (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:54:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ok2d2baqjscldo4vmovim4i18pgh-ephemeral-disk\", pod=\"cloud-network-config-controller-6b7699b59d-mtvjg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1aa0eaea-3879-4a75-b74f-75ef531c3b1d\"\n  Description: Pod ocm-arohcpci01-2t56ok2d2baqjscldo4vmovim4i18pgh-ephemeral-disk/cloud-network-config-controller-6b7699b59d-mtvjg (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:54:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56obh88cddik6iio0cu4ci35j02k7f-f1s2k3l8b2h6n5y\", pod=\"csi-snapshot-controller-7477f9489c-8hwvb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1a334069-0f8a-44c9-ab3a-5524390a74c8\"\n  Description: Pod ocm-arohcpci01-2t56obh88cddik6iio0cu4ci35j02k7f-f1s2k3l8b2h6n5y/csi-snapshot-controller-7477f9489c-8hwvb (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:54:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56obh88cddik6iio0cu4ci35j02k7f-f1s2k3l8b2h6n5y\", pod=\"cloud-network-config-controller-749db5479f-29nst\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ad67988d-3f68-4e02-84b2-957b828c23ea\"\n  Description: Pod ocm-arohcpci01-2t56obh88cddik6iio0cu4ci35j02k7f-f1s2k3l8b2h6n5y/cloud-network-config-controller-749db5479f-29nst (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:54:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ofblporgmut8brildeurh8bqh0jm-b8a2i8y2x4y0w5h\", pod=\"cloud-network-config-controller-79d7c84c4-jpmxx\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"350fe4f2-fba9-41ad-825a-5166b3b5b8f2\"\n  Description: Pod ocm-arohcpci01-2t56ofblporgmut8brildeurh8bqh0jm-b8a2i8y2x4y0w5h/cloud-network-config-controller-79d7c84c4-jpmxx (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:54:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56onftd04005nl721veuiicq8a5s6u-p1f3j3e2l1z4c9o\", pod=\"csi-snapshot-controller-5cc4dd7c9d-2tkdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4766f6e2-d1bc-40e5-9418-ada111a189ef\"\n  Description: Pod ocm-arohcpci01-2t56onftd04005nl721veuiicq8a5s6u-p1f3j3e2l1z4c9o/csi-snapshot-controller-5cc4dd7c9d-2tkdw (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:54:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56oclgsj9oapvabnc01b65dqdln9hs-q5s4p4g3c9i1p1b\", pod=\"csi-snapshot-controller-6445b45cbd-g46qp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4c46f8cc-e7a5-456d-88e9-4d97f220cbc0\"\n  Description: Pod ocm-arohcpci01-2t56oclgsj9oapvabnc01b65dqdln9hs-q5s4p4g3c9i1p1b/csi-snapshot-controller-6445b45cbd-g46qp (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:54:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56oguqit52ovjjoaakam9v60fven2c-sre-hcp-cluster\", pod=\"csi-snapshot-controller-68f64f94cf-f2r4v\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e4f7cb9e-9e38-408b-b971-4b23a9e6fc0e\"\n  Description: Pod ocm-arohcpci01-2t56oguqit52ovjjoaakam9v60fven2c-sre-hcp-cluster/csi-snapshot-controller-68f64f94cf-f2r4v (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 12:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:55:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56onftd04005nl721veuiicq8a5s6u-p1f3j3e2l1z4c9o\", pod=\"cloud-network-config-controller-c9f8f4988-98sd7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1b3a2719-6e7a-4a5b-aeae-d318e5e4d17e\"\n  Description: Pod ocm-arohcpci01-2t56onftd04005nl721veuiicq8a5s6u-p1f3j3e2l1z4c9o/cloud-network-config-controller-c9f8f4988-98sd7 (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 13:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:55:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ofgpftr9snm78jc1alvisvlihdkq-z9j9b2e5q2j7y1b\", pod=\"csi-snapshot-controller-7cf66cc49d-fnzp4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"6f42eca0-0eb8-412d-9fbb-e74e80100964\"\n  Description: Pod ocm-arohcpci01-2t56ofgpftr9snm78jc1alvisvlihdkq-z9j9b2e5q2j7y1b/csi-snapshot-controller-7cf66cc49d-fnzp4 (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 14:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:55:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56oclgsj9oapvabnc01b65dqdln9hs-q5s4p4g3c9i1p1b\", pod=\"csi-snapshot-controller-6445b45cbd-g46qp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4c46f8cc-e7a5-456d-88e9-4d97f220cbc0\"\n  Description: Pod ocm-arohcpci01-2t56oclgsj9oapvabnc01b65dqdln9hs-q5s4p4g3c9i1p1b/csi-snapshot-controller-6445b45cbd-g46qp (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 15:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:55:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56p42at8ondug5mpb3mmh6f9hh71ck-ea-listrk7c29\", pod=\"csi-snapshot-controller-ff4fdf4ff-8fn8s\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"523581cf-5012-43ac-a402-c7a0a8acaac4\"\n  Description: Pod ocm-arohcpci01-2t56p42at8ondug5mpb3mmh6f9hh71ck-ea-listrk7c29/csi-snapshot-controller-ff4fdf4ff-8fn8s (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 16:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:55:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56oguqit52ovjjoaakam9v60fven2c-sre-hcp-cluster\", pod=\"cloud-network-config-controller-85b777dd49-s25s6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"7f4bec03-7c59-4bee-bd1a-b303f9184e5c\"\n  Description: Pod ocm-arohcpci01-2t56oguqit52ovjjoaakam9v60fven2c-sre-hcp-cluster/cloud-network-config-controller-85b777dd49-s25s6 (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 17:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:55:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ofblporgmut8brildeurh8bqh0jm-b8a2i8y2x4y0w5h\", pod=\"cloud-network-config-controller-79d7c84c4-jpmxx\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"350fe4f2-fba9-41ad-825a-5166b3b5b8f2\"\n  Description: Pod ocm-arohcpci01-2t56ofblporgmut8brildeurh8bqh0jm-b8a2i8y2x4y0w5h/cloud-network-config-controller-79d7c84c4-jpmxx (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 18:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:56:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ofgpftr9snm78jc1alvisvlihdkq-z9j9b2e5q2j7y1b\", pod=\"csi-snapshot-controller-7cf66cc49d-fnzp4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"6f42eca0-0eb8-412d-9fbb-e74e80100964\"\n  Description: Pod ocm-arohcpci01-2t56ofgpftr9snm78jc1alvisvlihdkq-z9j9b2e5q2j7y1b/csi-snapshot-controller-7cf66cc49d-fnzp4 (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 19:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:56:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56oguqit52ovjjoaakam9v60fven2c-sre-hcp-cluster\", pod=\"cloud-network-config-controller-85b777dd49-s25s6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"7f4bec03-7c59-4bee-bd1a-b303f9184e5c\"\n  Description: Pod ocm-arohcpci01-2t56oguqit52ovjjoaakam9v60fven2c-sre-hcp-cluster/cloud-network-config-controller-85b777dd49-s25s6 (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 20:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:56:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ofgpftr9snm78jc1alvisvlihdkq-z9j9b2e5q2j7y1b\", pod=\"cloud-network-config-controller-c48d57456-gkqfz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"162d1556-afda-4dc1-97da-7279eda02fab\"\n  Description: Pod ocm-arohcpci01-2t56ofgpftr9snm78jc1alvisvlihdkq-z9j9b2e5q2j7y1b/cloud-network-config-controller-c48d57456-gkqfz (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 21:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:56:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56obh88cddik6iio0cu4ci35j02k7f-f1s2k3l8b2h6n5y\", pod=\"cloud-network-config-controller-749db5479f-29nst\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ad67988d-3f68-4e02-84b2-957b828c23ea\"\n  Description: Pod ocm-arohcpci01-2t56obh88cddik6iio0cu4ci35j02k7f-f1s2k3l8b2h6n5y/cloud-network-config-controller-749db5479f-29nst (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 22:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:56:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56oclgsj9oapvabnc01b65dqdln9hs-q5s4p4g3c9i1p1b\", pod=\"cloud-network-config-controller-5687d9df7d-mc789\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"c80807d8-f734-41fa-bf72-fa4544a1127f\"\n  Description: Pod ocm-arohcpci01-2t56oclgsj9oapvabnc01b65dqdln9hs-q5s4p4g3c9i1p1b/cloud-network-config-controller-5687d9df7d-mc789 (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 23:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:56:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ofblporgmut8brildeurh8bqh0jm-b8a2i8y2x4y0w5h\", pod=\"csi-snapshot-controller-546555d6dd-whrqm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"6e800052-3ed4-4832-ac6e-ba3a34c7eda1\"\n  Description: Pod ocm-arohcpci01-2t56ofblporgmut8brildeurh8bqh0jm-b8a2i8y2x4y0w5h/csi-snapshot-controller-546555d6dd-whrqm (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 24:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:56:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ok2d2baqjscldo4vmovim4i18pgh-ephemeral-disk\", pod=\"csi-snapshot-controller-68698d756-2sjcb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ea8389c4-6821-4c53-af9f-460b56d932d9\"\n  Description: Pod ocm-arohcpci01-2t56ok2d2baqjscldo4vmovim4i18pgh-ephemeral-disk/csi-snapshot-controller-68698d756-2sjcb (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 25:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:56:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ofgpftr9snm78jc1alvisvlihdkq-z9j9b2e5q2j7y1b\", pod=\"cloud-network-config-controller-c48d57456-gkqfz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"162d1556-afda-4dc1-97da-7279eda02fab\"\n  Description: Pod ocm-arohcpci01-2t56ofgpftr9snm78jc1alvisvlihdkq-z9j9b2e5q2j7y1b/cloud-network-config-controller-c48d57456-gkqfz (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 26:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:56:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56onftd04005nl721veuiicq8a5s6u-p1f3j3e2l1z4c9o\", pod=\"csi-snapshot-controller-5cc4dd7c9d-2tkdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4766f6e2-d1bc-40e5-9418-ada111a189ef\"\n  Description: Pod ocm-arohcpci01-2t56onftd04005nl721veuiicq8a5s6u-p1f3j3e2l1z4c9o/csi-snapshot-controller-5cc4dd7c9d-2tkdw (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 27:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:56:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ok2d2baqjscldo4vmovim4i18pgh-ephemeral-disk\", pod=\"csi-snapshot-controller-68698d756-2sjcb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ea8389c4-6821-4c53-af9f-460b56d932d9\"\n  Description: Pod ocm-arohcpci01-2t56ok2d2baqjscldo4vmovim4i18pgh-ephemeral-disk/csi-snapshot-controller-68698d756-2sjcb (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 28:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:56:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56oguqit52ovjjoaakam9v60fven2c-sre-hcp-cluster\", pod=\"csi-snapshot-controller-68f64f94cf-f2r4v\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e4f7cb9e-9e38-408b-b971-4b23a9e6fc0e\"\n  Description: Pod ocm-arohcpci01-2t56oguqit52ovjjoaakam9v60fven2c-sre-hcp-cluster/csi-snapshot-controller-68f64f94cf-f2r4v (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 29:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:56:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ofblporgmut8brildeurh8bqh0jm-b8a2i8y2x4y0w5h\", pod=\"csi-snapshot-controller-546555d6dd-whrqm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"6e800052-3ed4-4832-ac6e-ba3a34c7eda1\"\n  Description: Pod ocm-arohcpci01-2t56ofblporgmut8brildeurh8bqh0jm-b8a2i8y2x4y0w5h/csi-snapshot-controller-546555d6dd-whrqm (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 30:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:57:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56obh88cddik6iio0cu4ci35j02k7f-f1s2k3l8b2h6n5y\", pod=\"csi-snapshot-controller-7477f9489c-8hwvb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1a334069-0f8a-44c9-ab3a-5524390a74c8\"\n  Description: Pod ocm-arohcpci01-2t56obh88cddik6iio0cu4ci35j02k7f-f1s2k3l8b2h6n5y/csi-snapshot-controller-7477f9489c-8hwvb (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 31:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:57:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56p42at8ondug5mpb3mmh6f9hh71ck-ea-listrk7c29\", pod=\"cloud-network-config-controller-5c94ccc4b5-cg2cp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"dafd5d14-e303-47e0-b4e1-d01b00ea97f1\"\n  Description: Pod ocm-arohcpci01-2t56p42at8ondug5mpb3mmh6f9hh71ck-ea-listrk7c29/cloud-network-config-controller-5c94ccc4b5-cg2cp (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 32:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:57:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56oclgsj9oapvabnc01b65dqdln9hs-q5s4p4g3c9i1p1b\", pod=\"cloud-network-config-controller-5687d9df7d-mc789\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"c80807d8-f734-41fa-bf72-fa4544a1127f\"\n  Description: Pod ocm-arohcpci01-2t56oclgsj9oapvabnc01b65dqdln9hs-q5s4p4g3c9i1p1b/cloud-network-config-controller-5687d9df7d-mc789 (controller) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"dc1e25651421cdafde7ab4da463caef1ab2619d192be0d1a8d833f6246d3935a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"702288a89ce7dc61e45693f6260c580c01e0f296f382f69e2dd561371dbfa066","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000cd63f0\u003e: \n    resource group \"arm64-vm-cluster-62xw9fcdx42j\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-62xw9fcdx42j\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7d160b43-5f34-4632-80cb-5de8f2fd3f41\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7d160b43-5f34-4632-80cb-5de8f2fd3f41\",\n      \"name\": \"7d160b43-5f34-4632-80cb-5de8f2fd3f41\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:46:16.079175081Z\",\n      \"endTime\": \"2026-09-30T08:10:22.378932829Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ocn9um98ldcojp1pl53iclpb0kl7 still exists (deletion dispatched at 2026-09-30T07:46:30Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000cd63f0\u003e: \n    resource group \"arm64-vm-cluster-62xw9fcdx42j\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-62xw9fcdx42j\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7d160b43-5f34-4632-80cb-5de8f2fd3f41\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7d160b43-5f34-4632-80cb-5de8f2fd3f41\",\n      \"name\": \"7d160b43-5f34-4632-80cb-5de8f2fd3f41\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:46:16.079175081Z\",\n      \"endTime\": \"2026-09-30T08:10:22.378932829Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ocn9um98ldcojp1pl53iclpb0kl7 still exists (deletion dispatched at 2026-09-30T07:46:30Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e8da205e9aa80dcc7bf092793526ee503f73e2df50f20bd4637994d3a6dbc951","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"e0bad5b53413f07d5e84ab7f0e9ceb6826abb2c93e45e9f508ec37a3532918d0","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012cc090\u003e: \n    resource group \"ea-list-tcgjz7swjvdw\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listrk7c29\" in resourcegroup=\"ea-list-tcgjz7swjvdw\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19fe26bb-4a1e-4ebc-baa6-f57c53e83296\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19fe26bb-4a1e-4ebc-baa6-f57c53e83296\",\n      \"name\": \"19fe26bb-4a1e-4ebc-baa6-f57c53e83296\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:43:53.484653514Z\",\n      \"endTime\": \"2026-09-30T08:08:02.489234786Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56p42at8ondug5mpb3mmh6f9hh71ck still exists (deletion dispatched at 2026-09-30T07:44:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012cc090\u003e: \n    resource group \"ea-list-tcgjz7swjvdw\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listrk7c29\" in resourcegroup=\"ea-list-tcgjz7swjvdw\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19fe26bb-4a1e-4ebc-baa6-f57c53e83296\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19fe26bb-4a1e-4ebc-baa6-f57c53e83296\",\n      \"name\": \"19fe26bb-4a1e-4ebc-baa6-f57c53e83296\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:43:53.484653514Z\",\n      \"endTime\": \"2026-09-30T08:08:02.489234786Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56p42at8ondug5mpb3mmh6f9hh71ck still exists (deletion dispatched at 2026-09-30T07:44:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f44934dcd5dc7f5bed73669d671ede48170369c32f258b980f9017ea4e6e7399","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"f9a387d86ae70031b56c0d07a7a2eac9fb255e096d9d3c876eca81655334979d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0005468d0\u003e: \n    resource group \"rg-candidate-4-20-6zgdjl96fd2x\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-dn6n8z\" in resourcegroup=\"rg-candidate-4-20-6zgdjl96fd2x\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ec571ff-78ed-435f-b6c6-33518aa00501\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ec571ff-78ed-435f-b6c6-33518aa00501\",\n      \"name\": \"3ec571ff-78ed-435f-b6c6-33518aa00501\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:55:49.208034763Z\",\n      \"endTime\": \"2026-09-30T08:19:52.415025374Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ojtndhilnoii39q1v1t13gbjb4ah still exists (deletion dispatched at 2026-09-30T07:55:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0005468d0\u003e: \n    resource group \"rg-candidate-4-20-6zgdjl96fd2x\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-dn6n8z\" in resourcegroup=\"rg-candidate-4-20-6zgdjl96fd2x\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ec571ff-78ed-435f-b6c6-33518aa00501\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ec571ff-78ed-435f-b6c6-33518aa00501\",\n      \"name\": \"3ec571ff-78ed-435f-b6c6-33518aa00501\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:55:49.208034763Z\",\n      \"endTime\": \"2026-09-30T08:19:52.415025374Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ojtndhilnoii39q1v1t13gbjb4ah still exists (deletion dispatched at 2026-09-30T07:55:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f897c9adcc78f1e14a46b32c6431b2604a26ec3242433675cf0fb2e36d2d47fc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"92fe7c553dcdfef3768979dc519edf0e7995e69a064128d00e345d263790f82e","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001ed0138\u003e: \n    resource group \"rg-hs-pre--7p7rhfskdkg8\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-mxbwvc\" in resourcegroup=\"rg-hs-pre--7p7rhfskdkg8\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/38b5c072-1d7b-4021-9dde-2bef3cf5ae9a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/38b5c072-1d7b-4021-9dde-2bef3cf5ae9a\",\n      \"name\": \"38b5c072-1d7b-4021-9dde-2bef3cf5ae9a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:49:12.08839353Z\",\n      \"endTime\": \"2026-09-30T08:13:12.35854499Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56obncvdr9t1vsvq3snfkacnvh3cke still exists (deletion dispatched at 2026-09-30T07:49:18Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001ed0138\u003e: \n    resource group \"rg-hs-pre--7p7rhfskdkg8\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-mxbwvc\" in resourcegroup=\"rg-hs-pre--7p7rhfskdkg8\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/38b5c072-1d7b-4021-9dde-2bef3cf5ae9a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/38b5c072-1d7b-4021-9dde-2bef3cf5ae9a\",\n      \"name\": \"38b5c072-1d7b-4021-9dde-2bef3cf5ae9a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:49:12.08839353Z\",\n      \"endTime\": \"2026-09-30T08:13:12.35854499Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56obncvdr9t1vsvq3snfkacnvh3cke still exists (deletion dispatched at 2026-09-30T07:49:18Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":35,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","293006e7a1ad208844fa10f4e2b3c0c8959662e799277a9c1d23e6534078c9e0","392750d533ddeabf76d5209f9e2a09e5f48e64ee98fce8c13f1c6bd36af29cfa","433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","c65eaecf2bcf9cab816e9e9171a4f334676466d8dfb2f88bb604984fc8e6cfe2","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","f570ff2ef05014779bb7489dca295e256a69aead33d6bff8319ed54db32fd9e1","fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1"],"matched_occurrences":35,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105186118872862720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7250,"pr_state":"open","pr_sha":"96ca57cb7d5eab9eb9db1b06eaec792e6f8685b8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T06:40:57Z","started_at":"2026-09-30T06:40:57Z","completed_at":"2026-09-30T08:59:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T09:11:41.312283573Z","timing_metadata_checked_at":"2026-09-30T09:11:41.312283573Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T09:11:41.316143062Z","region_metadata_checked_at":"2026-09-30T09:11:41.316143062Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105177784723705856","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7170,"pr_state":"closed","pr_sha":"715e1ab64227414bccc3c04d703bb2672c050aea","final_merged_sha":"715e1ab64227414bccc3c04d703bb2672c050aea","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T06:07:50Z","started_at":"2026-09-30T06:07:50Z","completed_at":"2026-09-30T08:27:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T08:40:21.848278244Z","timing_metadata_checked_at":"2026-09-30T08:40:21.848278244Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T08:40:21.851281134Z","region_metadata_checked_at":"2026-09-30T08:40:21.851281134Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7258/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105170147533656064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7258,"pr_state":"closed","pr_sha":"fc6fe3e052e452bc85be384a80e0c231a4ac5b9b","final_merged_sha":"a68022814b830b9a8334e00ff48929e4987b9c4a","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T05:37:29Z","started_at":"2026-09-30T05:37:29Z","completed_at":"2026-09-30T07:52:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T08:09:08.404045017Z","timing_metadata_checked_at":"2026-09-30T08:09:08.404045017Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T08:09:08.407668906Z","region_metadata_checked_at":"2026-09-30T08:09:08.407668906Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105167861478592512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7181,"pr_state":"closed","pr_sha":"541a1eba3ec70c4d9c21dacb6cc888e561ca9f11","final_merged_sha":"3ae51eab9b946eb1b94407de9646f52a67175cd0","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T05:28:24Z","started_at":"2026-09-30T05:28:24Z","completed_at":"2026-09-30T07:39:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T07:58:09.691706225Z","timing_metadata_checked_at":"2026-09-30T07:58:09.691706225Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T07:58:09.694735017Z","region_metadata_checked_at":"2026-09-30T07:58:09.694735017Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"4ffb183209c2b567847865a5e816fc23a41d17b1785cf246dc8ab93f2041bf84","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105167861478592512","occurred_at":"2026-09-30T05:28:24Z","failed_at":"e2e","signature_id":"241350474584925c505b6db759b0c1f71619ba1d16c84a08ca9783ed5efd6581","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:487]: ClusterVersion had no completed update entry in history\nExpected\n    \u003cstring\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:487]: ClusterVersion had no completed update entry in history\nExpected\n    \u003cstring\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"579aca9a055614c4c372f82f6c1cd8ec85f037f92e539fb50ce4ea435c084257","failure_pattern":"ClusterVersion had no completed update entry in history","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:487]: ClusterVersion had no completed update entry in history"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["579aca9a055614c4c372f82f6c1cd8ec85f037f92e539fb50ce4ea435c084257"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7158/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105165316576251904","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7158,"pr_state":"open","pr_sha":"ad4b8d135c2f719ba56447c9417bd2d0c51dbf22","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T05:18:18Z","started_at":"2026-09-30T05:18:18Z","completed_at":"2026-09-30T07:34:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T07:47:14.865476292Z","timing_metadata_checked_at":"2026-09-30T07:47:14.865476292Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T07:47:14.869828678Z","region_metadata_checked_at":"2026-09-30T07:47:14.869828678Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"5c5a9007f6743cd240206942b466c26c1d9fc804d090e4a923afbab1e6608ea1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7158/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105165316576251904","occurred_at":"2026-09-30T05:18:18Z","failed_at":"alert","signature_id":"0e4d81750c983a9cda845d641cdc28ac3126144f9f77408e60f4a30101f8fb9b","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T06:42:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T06:42:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-30T06:49:03Z\n  Ended:       2026-09-30T07:01:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"klusterlet-2t55he6nad38adkphki745bqugvhhgmi\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment klusterlet-2t55he6nad38adkphki745bqugvhhgmi/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-30T06:49:04Z\n  Ended:       2026-09-30T07:01:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"klusterlet-2t55he6nad38adkphki745bqugvhhgmi\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment klusterlet-2t55he6nad38adkphki745bqugvhhgmi/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired"}},{"row_id":"78f5d749ba47b252b6e96b7f7cbe61dbe94c533829e8931a0126e9dbb96a39c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7158/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105165316576251904","occurred_at":"2026-09-30T05:18:18Z","failed_at":"alert","signature_id":"f24999767c880e9e7cc09f5f63e3720ef83e3427a0a79dedbcfca71bdb5f3dae","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T06:07:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-78648d9c8d-czf5c\", region=\"westus3\", severity=\"warning\"\n  Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-78648d9c8d-czf5c has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"846903df095efb338b6f1d9ffe27dc9dfb97c4eb67a5730e4fa51bcc62c61082","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7158/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105165316576251904","occurred_at":"2026-09-30T05:18:18Z","failed_at":"alert","signature_id":"e449da77403f611870944fef6e8e8243f367e4154d6925c4af4f6d5765343341","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 12 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T06:32:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T06:32:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-30T06:39:00Z\n  Ended:       2026-09-30T07:01:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"klusterlet-2t55he6nad38adkphki745bqugvhhgmi\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t55he6nad38adkphki745bqugvhhgmi/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-30T06:39:00Z\n  Ended:       2026-09-30T07:01:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"klusterlet-2t55he6nad38adkphki745bqugvhhgmi\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t55he6nad38adkphki745bqugvhhgmi/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-30T07:02:01Z\n  Ended:       2026-09-30T07:08:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"klusterlet-2t55s5ip2mfud3dnigkagbgr229suldb\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t55s5ip2mfud3dnigkagbgr229suldb/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-30T07:02:01Z\n  Ended:       2026-09-30T07:08:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"klusterlet-2t55sahljftd8u6q1m4sbae73q1qo4f2\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t55sahljftd8u6q1m4sbae73q1qo4f2/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-09-30T07:02:01Z\n  Ended:       2026-09-30T07:08:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"klusterlet-2t55s5ip2mfud3dnigkagbgr229suldb\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t55s5ip2mfud3dnigkagbgr229suldb/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-09-30T07:02:01Z\n  Ended:       2026-09-30T07:08:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"klusterlet-2t55sahljftd8u6q1m4sbae73q1qo4f2\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t55sahljftd8u6q1m4sbae73q1qo4f2/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-09-30T07:03:02Z\n  Ended:       2026-09-30T07:12:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"klusterlet-2t55soh6c16uoks7fn5rqknhr3g6cc88\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t55soh6c16uoks7fn5rqknhr3g6cc88/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-09-30T07:03:02Z\n  Ended:       2026-09-30T07:12:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"klusterlet-2t55soh6c16uoks7fn5rqknhr3g6cc88\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t55soh6c16uoks7fn5rqknhr3g6cc88/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-09-30T07:03:02Z\n  Ended:       2026-09-30T07:11:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"klusterlet-2t55som67oi1sqntaq9acgv9gavh7vjf\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t55som67oi1sqntaq9acgv9gavh7vjf/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-09-30T07:03:02Z\n  Ended:       2026-09-30T07:11:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"klusterlet-2t55som67oi1sqntaq9acgv9gavh7vjf\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t55som67oi1sqntaq9acgv9gavh7vjf/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7229/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105159944369082368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7229,"pr_state":"closed","pr_sha":"e251feee783228a94dd7ed8590b63c06c237cbb6","final_merged_sha":"e251feee783228a94dd7ed8590b63c06c237cbb6","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T04:56:57Z","started_at":"2026-09-30T04:56:57Z","completed_at":"2026-09-30T07:11:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T07:26:23.763459131Z","timing_metadata_checked_at":"2026-09-30T07:26:23.763459131Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T07:26:23.766679221Z","region_metadata_checked_at":"2026-09-30T07:26:23.766679221Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"08692a2a9bb993dc523ebc6bad950528d9a4e4e132cf9f5c58cb36f896caa8a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7229/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105159944369082368","occurred_at":"2026-09-30T04:56:57Z","failed_at":"e2e","signature_id":"b7b881f56be7fc493c98503286f3b339d22b05150a32a7c2547bdf5dad7f2da5","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-78dt7bb9qdx9\" to finish creating: GET https://rp.j9082368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3f34a8c-014f-4784-8b2d-d242f6bfcb85\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeadlineExceeded\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3f34a8c-014f-4784-8b2d-d242f6bfcb85\",\n  \"name\": \"e3f34a8c-014f-4784-8b2d-d242f6bfcb85\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-30T05:49:26.691969203Z\",\n  \"endTime\": \"2026-09-30T06:08:29.964580175Z\",\n  \"error\": {\n    \"code\": \"DeadlineExceeded\",\n    \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine t1n4p5j2m9y1n6b-np-update-nodes-fvh6z-s276d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-78dt7bb9qdx9--managed/providers/Microsoft.Compute/virtualMachines/t1n4p5j2m9y1n6b-np-update-nodes-fvh6z-s276d to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine t1n4p5j2m9y1n6b-np-update-nodes-fvh6z-s276d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-78dt7bb9qdx9--managed/providers/Microsoft.Compute/virtualMachines/t1n4p5j2m9y1n6b-np-update-nodes-fvh6z-s276d to exist\\n\"\n  }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n    \u003c[]error | len:1, cap:1\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-78dt7bb9qdx9\" to finish creating: GET https://rp.j9082368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3f34a8c-014f-4784-8b2d-d242f6bfcb85\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeadlineExceeded\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3f34a8c-014f-4784-8b2d-d242f6bfcb85\",\n  \"name\": \"e3f34a8c-014f-4784-8b2d-d242f6bfcb85\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-30T05:49:26.691969203Z\",\n  \"endTime\": \"2026-09-30T06:08:29.964580175Z\",\n  \"error\": {\n    \"code\": \"DeadlineExceeded\",\n    \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine t1n4p5j2m9y1n6b-np-update-nodes-fvh6z-s276d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-78dt7bb9qdx9--managed/providers/Microsoft.Compute/virtualMachines/t1n4p5j2m9y1n6b-np-update-nodes-fvh6z-s276d to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine t1n4p5j2m9y1n6b-np-update-nodes-fvh6z-s276d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-78dt7bb9qdx9--managed/providers/Microsoft.Compute/virtualMachines/t1n4p5j2m9y1n6b-np-update-nodes-fvh6z-s276d to exist\\n\"\n  }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n    \u003c[]error | len:1, cap:1\u003e: ...\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5","failure_pattern":"--------------------------------------------------------------------------------","search_query":"--------------------------------------------------------------------------------"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105143371457630208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7170,"pr_state":"closed","pr_sha":"715e1ab64227414bccc3c04d703bb2672c050aea","final_merged_sha":"715e1ab64227414bccc3c04d703bb2672c050aea","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T03:51:06Z","started_at":"2026-09-30T03:51:06Z","completed_at":"2026-09-30T04:10:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T04:25:33.371744093Z","timing_metadata_checked_at":"2026-09-30T04:25:33.371744093Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-30T04:25:33.375447782Z","region_metadata_checked_at":"2026-09-30T05:28:39.014161271Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"68641540efce75138e29794b3953a507c41d24802b873009a748bc5e5e2908bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105143371457630208","occurred_at":"2026-09-30T03:51:06Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105143374678855680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7244,"pr_state":"closed","pr_sha":"cbddf39100b6fb0f90835d16a76eb4cb925cc7df","final_merged_sha":"cbddf39100b6fb0f90835d16a76eb4cb925cc7df","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T03:51:06Z","started_at":"2026-09-30T03:51:06Z","completed_at":"2026-09-30T06:11:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T06:23:57.279533434Z","timing_metadata_checked_at":"2026-09-30T06:23:57.279533434Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T06:23:57.282751024Z","region_metadata_checked_at":"2026-09-30T06:23:57.282751024Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7238/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105143304160022528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7238,"pr_state":"closed","pr_sha":"6a1ffdb2da1b44e3e97762eec9c6e4bac7581c86","final_merged_sha":"6a1ffdb2da1b44e3e97762eec9c6e4bac7581c86","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T03:50:49Z","started_at":"2026-09-30T03:50:49Z","completed_at":"2026-09-30T06:15:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T06:34:35.190711072Z","timing_metadata_checked_at":"2026-09-30T06:34:35.190711072Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T06:34:35.193966462Z","region_metadata_checked_at":"2026-09-30T06:34:35.193966462Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105132902411931648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6970,"pr_state":"closed","pr_sha":"af55f114edf1b1d398480eeee6a41ae706e821e2","final_merged_sha":"af55f114edf1b1d398480eeee6a41ae706e821e2","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T03:09:30Z","started_at":"2026-09-30T03:09:30Z","completed_at":"2026-09-30T05:36:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T05:49:44.6045911Z","timing_metadata_checked_at":"2026-09-30T05:49:44.6045911Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T05:49:44.60781189Z","region_metadata_checked_at":"2026-09-30T05:49:44.60781189Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"52c6504f3dce91cfdbfdd66a274d0e4263e289333775f4141281bf373e24ce03","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105132902411931648","occurred_at":"2026-09-30T03:09:30Z","failed_at":"e2e","signature_id":"93fbf55b219b33dbed79b2f7d0789e2d4a2fce95865a645fcfba661c8688665a","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010db2a0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-gwhcb7q77s5g\" to finish creating: GET https://rp.j1931648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e26eb862-dd0d-4748-bf3d-aaa27c84e026\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e26eb862-dd0d-4748-bf3d-aaa27c84e026\",\n      \"name\": \"e26eb862-dd0d-4748-bf3d-aaa27c84e026\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T04:22:37.324871209Z\",\n      \"endTime\": \"2026-09-30T04:41:46.747553014Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine l4b9n5d1w5j2a3f-autoscale-noaz-wmbj7-vtxcx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-gwhcb7q77s5g--managed/providers/Microsoft.Compute/virtualMachines/l4b9n5d1w5j2a3f-autoscale-noaz-wmbj7-vtxcx to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine l4b9n5d1w5j2a3f-autoscale-noaz-wmbj7-vtxcx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-gwhcb7q77s5g--managed/providers/Microsoft.Compute/virtualMachines/l4b9n5d1w5j2a3f-autoscale-noaz-wmbj7-vtxcx to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010db2a0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-gwhcb7q77s5g\" to finish creating: GET https://rp.j1931648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e26eb862-dd0d-4748-bf3d-aaa27c84e026\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e26eb862-dd0d-4748-bf3d-aaa27c84e026\",\n      \"name\": \"e26eb862-dd0d-4748-bf3d-aaa27c84e026\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T04:22:37.324871209Z\",\n      \"endTime\": \"2026-09-30T04:41:46.747553014Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine l4b9n5d1w5j2a3f-autoscale-noaz-wmbj7-vtxcx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-gwhcb7q77s5g--managed/providers/Microsoft.Compute/virtualMachines/l4b9n5d1w5j2a3f-autoscale-noaz-wmbj7-vtxcx to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine l4b9n5d1w5j2a3f-autoscale-noaz-wmbj7-vtxcx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-gwhcb7q77s5g--managed/providers/Microsoft.Compute/virtualMachines/l4b9n5d1w5j2a3f-autoscale-noaz-wmbj7-vtxcx to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105112031697833984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7054,"pr_state":"closed","pr_sha":"7553575593411fb5f3c1d57088aa752a234c76d1","final_merged_sha":"7553575593411fb5f3c1d57088aa752a234c76d1","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T01:46:34Z","started_at":"2026-09-30T01:46:34Z","completed_at":"2026-09-30T04:01:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T04:15:05.670308444Z","timing_metadata_checked_at":"2026-09-30T04:15:05.670308444Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T04:15:05.673626135Z","region_metadata_checked_at":"2026-09-30T04:15:05.673626135Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7229/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105107374309117952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7229,"pr_state":"closed","pr_sha":"e251feee783228a94dd7ed8590b63c06c237cbb6","final_merged_sha":"e251feee783228a94dd7ed8590b63c06c237cbb6","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T01:28:03Z","started_at":"2026-09-30T01:28:03Z","completed_at":"2026-09-30T03:46:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T04:04:20.674529776Z","timing_metadata_checked_at":"2026-09-30T04:04:20.674529776Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T04:04:20.67949106Z","region_metadata_checked_at":"2026-09-30T04:04:20.67949106Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"530dc82e6293a2d2ae7f02cd41370127a1f364dc8a592f851afe21243fc02284","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7229/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105107374309117952","occurred_at":"2026-09-30T01:28:03Z","failed_at":"e2e","signature_id":"a820df5e90389c4dc2a4194c6adffcc807b799a735d1bfbc28265e77991fcf6b","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:130]: failed to create customer resources for aggregated feature cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009224e0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-agg-cluster-kv5cgd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-agg-cluster-kv5cgd\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: DeploymentNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DeploymentNotFound\",\n        \"message\": \"Deployment 'mi-agg-cluster-kv5cgd' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:130]: failed to create customer resources for aggregated feature cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009224e0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-agg-cluster-kv5cgd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-agg-cluster-kv5cgd\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: DeploymentNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DeploymentNotFound\",\n        \"message\": \"Deployment 'mi-agg-cluster-kv5cgd' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"72716b475dfdb36ca6de5b4a7866bbc958e1921fb2c6ea0b3467c231190cd12c","failure_pattern":"ERROR CODE: DeploymentNotFound","search_query":"ERROR CODE: DeploymentNotFound"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["72716b475dfdb36ca6de5b4a7866bbc958e1921fb2c6ea0b3467c231190cd12c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105106955633692672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6775,"pr_state":"open","pr_sha":"02cba68fff12e77be3d865d9e0d24a307718ee82","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T01:26:23Z","started_at":"2026-09-30T01:26:23Z","completed_at":"2026-09-30T03:42:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T03:53:37.014712713Z","timing_metadata_checked_at":"2026-09-30T03:53:37.014712713Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T03:53:37.0189678Z","region_metadata_checked_at":"2026-09-30T03:53:37.0189678Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105092336139636736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7001,"pr_state":"closed","pr_sha":"3c72ea92559bff30660a213a1ce63775f8a085c7","final_merged_sha":"3c72ea92559bff30660a213a1ce63775f8a085c7","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T00:28:18Z","started_at":"2026-09-30T00:28:18Z","completed_at":"2026-09-30T02:47:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T03:01:35.899144616Z","timing_metadata_checked_at":"2026-09-30T03:01:35.899144616Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T03:01:35.902808904Z","region_metadata_checked_at":"2026-09-30T03:01:35.902808904Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}